Windows

Output video data

1. To enable the raw video data callback, call the following interface:

int EnableVideoFrameObserver(bool enabled, unsigned int position);

2. Register a listener for the raw video data callback as follows:

int RegisterVideoFrameObserver(RtcEngineVideoFrameObserver* observer)

3. Retrieve raw video data through the RtcEngineVideoFrameObserver callback as follows:

/**
* @brief Video format capture preference.
*/
RtcEngineVideoPixelFormat GetVideoFormatPreference();

/**
* @brief Local captured video data.
* @param frame video sample
* @return 
* - true: The data needs to be written back to the SDK (default).
* - false: The data does not need to be written back to the SDK.
*/
bool OnCaptureVideoFrame(RtcEngineVideoFrame &frame);

/**
* @brief Local video data before encoding.
* @param videoTrack video track
* @param frame video sample
* @return 
* - true: The data needs to be written back to the SDK (default).
* - false: The data does not need to be written back to the SDK.
*/
bool OnPreEncodeVideoFrame(RtcEngineVideoTrack videoTrack, RtcEngineVideoFrame &frame);

/**
* @brief Remote video data after decoding.
* @param userId user id
* @param videoTrack video track
* @param frame video sample
* @return 
* - true: The data needs to be written back to the SDK (default).
* - false: The data does not need to be written back to the SDK.
*/
bool OnRemoteVideoFrame(ding::rtc::String userId, RtcEngineVideoTrack videoTrack, RtcEngineVideoFrame &frame);

4. The following sample code shows how to write subscribed raw remote video data to a local YUV file:

class DemoVideoObserver : public ding::rtc::RtcEngineVideoFrameObserver
{
public:
  ~DemoVideoObserver();
  ding::rtc::RtcEngineVideoPixelFormat GetVideoFormatPreference() override {
    return ding::rtc::RtcEngineVideoPixelFormat::RtcEngineVideoI420;
  }

  bool OnCaptureVideoFrame(ding::rtc::RtcEngineVideoFrame &frame) override {
    std::string streamId = "capture";
    SaveToFile(streamId, frame);
    return false;
  }

  bool OnRemoteVideoFrame(ding::rtc::String uid,
    ding::rtc::RtcEngineVideoTrack track,
    ding::rtc::RtcEngineVideoFrame &frame) override
  {
    std::string streamId = std::string(uid.c_str()) + "-" + (track == ding::rtc::RtcEngineVideoTrack::RtcEngineVideoTrackCamera ? "camera" : "screen");
    SaveToFile(streamId, frame);
    return false;
  }

  bool OnPreEncodeVideoFrame(
    ding::rtc::RtcEngineVideoTrack track,
    ding::rtc::RtcEngineVideoFrame &frame) override
  {
    std::string streamId = std::string("publish") + "-" + (track == ding::rtc::RtcEngineVideoTrack::RtcEngineVideoTrackCamera ? "camera" : "screen");
    SaveToFile(streamId, frame);
    return false;
  }

  // to save disk usage, scale down to 320x240, max 200 frames
  class StreamInfo {
  public:
    std::string streamId; // uid # tracktype
    int framesSaved_;
    FILE *fp_;
  };
  void SaveToFile(const std::string &fileName, ding::rtc::RtcEngineVideoFrame &frame);

  std::map<std::string, StreamInfo> streamMap_; // guarded by mutex_
  std::mutex mutex_;
};

DemoVideoObserver::~DemoVideoObserver()
{
    for (auto it=streamMap_.begin(); it!=streamMap_.end(); it++) {
        fclose(it->second.fp_);
    }
}

void DemoVideoObserver::SaveToFile(const std::string &streamId,
    ding::rtc::RtcEngineVideoFrame &frame)
{
    bool found = false;
    mutex_.lock();
    if (streamMap_.find(streamId) != streamMap_.end()) {
        found = true;
    }
    mutex_.unlock();

    if (!found) {
        std::string channel = GetGlobalContext()->dc_.appserver_param_.channelName;
        std::string dir = GetGlobalContext()->dc_.utf8_working_path + "/record/" + channel;
        recursive_mkdir(dir.c_str());

        std::string fileName = dir + "/" + streamId + "-320x240.yuv";
        FILE *fp = fopen(fileName.c_str(), "wb");
        if (fp == NULL) {
            printf("failed to create file %s\n", fileName.c_str());
            return;
        }

        StreamInfo si;
        si.streamId = streamId;
        si.framesSaved_ = 0;
        si.fp_ = fp;

        mutex_.lock();
        streamMap_[streamId] = si;
        mutex_.unlock();
    }

    mutex_.lock();
    FILE *fp = streamMap_[streamId].fp_;
    int count = streamMap_[streamId].framesSaved_;
    mutex_.unlock();

    if (frame.frameType != ding::rtc::RtcEngineVideoFrameType::RtcEngineVideoFrameRaw) {
        return;
    }

    if (frame.pixelFormat != ding::rtc::RtcEngineVideoPixelFormat::RtcEngineVideoI420) {
        return;
    }


    // scale down to 320x240
    ding::rtc::RtcEngineVideoFrame tmp;
    memset(&tmp, 0, sizeof(ding::rtc::RtcEngineVideoFrame));
    tmp.frameType = ding::rtc::RtcEngineVideoFrameType::RtcEngineVideoFrameRaw;
    tmp.pixelFormat = ding::rtc::RtcEngineVideoPixelFormat::RtcEngineVideoI420;
    tmp.width = 320;
    tmp.height = 240;
    int ystride = (tmp.width + 1) /  2 * 2;
    int hstride = (tmp.height + 1) / 2 * 2;
    tmp.stride[0] = ystride;
    tmp.stride[1] = ystride / 2;
    tmp.stride[2] = ystride / 2;
    tmp.offset[0] = 0;
    tmp.offset[1] = ystride * hstride;
    tmp.offset[2] = ystride * hstride * 5 / 4;
    int size = ystride * hstride * 3 / 2;
    tmp.data = malloc(size);

    const uint8_t *src[3];
    src[0] = (uint8_t *)frame.data + frame.offset[0];
    src[1] = (uint8_t *)frame.data + frame.offset[1];
    src[2] = (uint8_t *)frame.data + frame.offset[2];
    uint32_t src_strides[3];
    src_strides[0] = frame.stride[0];
    src_strides[1] = frame.stride[1];
    src_strides[2] = frame.stride[2];
    uint8_t *dst[3];
    dst[0] = (uint8_t *)tmp.data + tmp.offset[0];
    dst[1] = (uint8_t *)tmp.data + tmp.offset[1];
    dst[2] = (uint8_t *)tmp.data + tmp.offset[2];
    uint32_t dst_strides[3];
    dst_strides[0] = tmp.stride[0];
    dst_strides[1] = tmp.stride[1];
    dst_strides[2] = tmp.stride[2];
    ResizeYuvBuffers(src, src_strides,
        frame.width, frame.height,
        dst, dst_strides,
        tmp.width, tmp.height);

    ystride = tmp.stride[0];
    hstride = (tmp.height + 1) / 2 * 2;

    // y buffer
    size = ystride * hstride;
    const uint8_t *y = (const uint8_t *)tmp.data + tmp.offset[0];
    fwrite(y, 1, size, fp);

    // u buffer
    size = size / 4;
    const uint8_t *u = (const uint8_t *)tmp.data + tmp.offset[1];
    fwrite(u, 1, size, fp);

    // v buffer
    const uint8_t *v = (const uint8_t *)tmp.data + tmp.offset[2];
    fwrite(v, 1, size, fp);

    mutex_.lock();
    streamMap_[streamId].framesSaved_++;
    mutex_.unlock();

    free(tmp.data);
}

Output raw audio data

1. Enable the callback listener for raw audio data:

int EnableAudioFrameObserver(bool enabled, unsigned int position, const RtcEngineAudioFrameObserverConfig& config)

2. Register the callback listener for raw audio data:

int RegisterAudioFrameObserver(RtcEngineAudioFrameObserver* observer);

3. Retrieve raw audio data through the RtcEngineAudioFrameObserver callback as follows:

@optional
/**
 * @brief Callback for remote stream pulling data.
 * @param uid The remote user ID.
 * @param frame Audio data. For more information, see {@link RtcEngineAudioFrame}.
 * @note Do not perform any time-consuming operations in this callback function. Otherwise, audio may become abnormal.
 */
- (void)onRemoteUserAudioFrame:(NSString *_Nonnull)uid frame:(DingRtcAudioDataSample * _Nonnull)frame;

/**
 * @brief Callback for locally subscribed audio data.
 * @details Audio data from all remote users mixed for playback.
 * @param frame Audio data.
*/
- (void)onPlaybackAudioFrame:(DingRtcAudioDataSample * _Nonnull)frame;

/**
 * @brief Callback for locally captured audio data.
 * @param frame Audio data.
 */
- (void)onCapturedAudioFrame:(DingRtcAudioDataSample * _Nonnull)frame;

/**
 * @brief Callback for data after 3A processing.
 * @param frame Audio data.
*/
- (void)onProcessCapturedAudioFrame:(DingRtcAudioDataSample * _Nonnull)frame;

/**
 * @brief Callback for local audio data for stream ingest.
 * @param frame Audio data.
 */
- (void)onPublishAudioFrame:(DingRtcAudioDataSample * _Nonnull)frame;

@end

4. The following sample code shows how to subscribe to raw remote audio data and write it to a local PCM file:

class DemoAudioSink : public ding::rtc::RtcEngineAudioFrameObserver
{
public:
	DemoAudioSink();
    ~DemoAudioSink();

	void OnPlaybackAudioFrame(ding::rtc::RtcEngineAudioFrame &frame) override;

    void OnCapturedAudioFrame(ding::rtc::RtcEngineAudioFrame &frame) override;

    void OnProcessCapturedAudioFrame(ding::rtc::RtcEngineAudioFrame &frame) override;

    void OnPublishAudioFrame(ding::rtc::RtcEngineAudioFrame &frame) override;

  	void OnRemoteUserAudioFrame(const char *uid, ding::rtc::RtcEngineAudioFrame &frame) override;

private:
    void saveOneFrame(const std::string &id, ding::rtc::RtcEngineAudioFrame &frame);

    int64_t max_frames_;
    struct audio_track_record_context {
     	FILE *fp;
      	bool file_error;
      	int64_t frame_count;
    };
    std::map<std::string, struct audio_track_record_context> record_contexts;
};


DemoAudioSink::DemoAudioSink()
{
  max_frames_ = 50 * 30;
}

DemoAudioSink::~DemoAudioSink()
{
  for (auto it = record_contexts.begin(); it != record_contexts.end(); it++) {
    if (it->second.fp != NULL) {
      fclose(it->second.fp);
    }
  }
}

void DemoAudioSink::saveOneFrame(const std::string &id, ding::rtc::RtcEngineAudioFrame &frame)
{
    if (record_contexts.find(id) == record_contexts.end()) {
        struct audio_track_record_context empty;
        empty.fp = NULL;
        empty.file_error = false;
        empty.frame_count = 0;
        record_contexts[id] = empty;
    }
    if (record_contexts[id].frame_count >= max_frames_) {
        return;
    }

    if (record_contexts[id].file_error) {
        return;
    }

    if (record_contexts[id].fp == NULL) {
        std::string file;
        std::string channel = GetGlobalContext()->dc_.appserver_param_.channelName;
        std::string dir = GetGlobalContext()->dc_.utf8_working_path + "/record/" + channel;
        recursive_mkdir(dir.c_str());
        char t[128];
        sprintf(t, "%d_%s_%s.pcm", frame.samplesPerSec, frame.channels == 2 ? "stereo" : "mono", id.c_str());
        file = dir + "/" + t;

        record_contexts[id].fp = fopen(file.c_str(), "wb");
        if (record_contexts[id].fp == NULL) {
            printf("failed to create file %s\n", file.c_str());
            record_contexts[id].file_error = true;
            return;
        }
        else {
            printf("created file %s\n", file.c_str());
        }
    }

    if (frame.type != ding::rtc::RtcEngineAudioFrameType::RtcEngineAudioFramePcm16) {
        return;
    }

    size_t size = frame.samples * frame.bytesPerSample * frame.channels;
    if (fwrite(frame.buffer, 1, size, record_contexts[id].fp) != size) {
        record_contexts[id].file_error = true;
        fclose(record_contexts[id].fp);
        record_contexts[id].fp = NULL;
        return;
    }

    record_contexts[id].frame_count++;
}

void DemoAudioSink::OnRemoteUserAudioFrame(const char *uid, ding::rtc::RtcEngineAudioFrame &frame)
{
    saveOneFrame(uid, frame);
}

void DemoAudioSink::OnPlaybackAudioFrame(ding::rtc::RtcEngineAudioFrame &frame)
{
    saveOneFrame("playback", frame);
}

void DemoAudioSink::OnCapturedAudioFrame(ding::rtc::RtcEngineAudioFrame &frame)
{
    saveOneFrame("capture", frame);
}

void DemoAudioSink::OnProcessCapturedAudioFrame(ding::rtc::RtcEngineAudioFrame &frame)
{
    saveOneFrame("processed", frame);
}

void DemoAudioSink::OnPublishAudioFrame(ding::rtc::RtcEngineAudioFrame &frame)
{
    saveOneFrame("publish", frame);
}