DocsICP FilingConsole
官方文档
首页

Get raw audio data

更新时间：
复制 MD 格式
产品详情
我的收藏

Get captured raw audio data from the ARTC SDK for custom processing.

Sample code

Get raw audio data on Android: Android/ARTCExample/AdvancedUsage/src/main/java/com/aliyun/artc/api/advancedusage/ProcessAudioRawData/ProcessAudioRawDataActivity.java.

Get raw audio data on iOS: iOS/ARTCExample/AdvancedUsage/ProcessAudioRawData/ProcessAudioRawDataVC.swift.

Get raw audio data on Harmony: Harmony/ARTCExample/entry/src/main/ets/pages/advancedusage/ProcessAudioRawDataPage.ets.

Prerequisites

Ensure that you meet the following requirements:

Procedure

image

1. Initialize the ARTC engine

Create and initialize an ARTC engine instance before getting raw audio data.

2. Implement audio data callbacks

Implement the appropriate callback methods to get audio data. The following table lists the available callbacks.

Callback

Description

Notes

onCapturedAudioFrame

Callback for locally captured audio data.

None.

onProcessCapturedAudioFrame

Callback for audio data after it is processed by the 3A algorithms.

None.

onPublishAudioFrame

Callback for the audio data being published.

Can only be set to read-only mode.

onPlaybackAudioFrame

Callback for the audio data being played locally, which may include mixed audio from multiple remote users.

None.

onRemoteUserAudioFrame

Callback for the audio data of a specific remote user.

You cannot set the sample rate or channel count because they follow the settings of the remote user. However, you can set the mode to read-write.

Android

Implement the required methods of the AliRtcAudioFrameObserver class.

private final AliRtcEngine.AliRtcAudioFrameObserver rtcAudioFrameObserver = new AliRtcEngine.AliRtcAudioFrameObserver() {
    @Override
    public boolean onCapturedAudioFrame(AliRtcEngine.AliRtcAudioFrame frame) {
        // Callback for locally captured audio data.
        Log.i(TAG, "onCaptureAudioFrame");
        return true;
    }

    @Override
    public boolean onProcessCapturedAudioFrame(AliRtcEngine.AliRtcAudioFrame frame) {
        // Callback for audio data after 3A processing.
        Log.i(TAG, "onProcessCaptureAudioFrame");
        return true;
    }

    @Override
    public boolean onPublishAudioFrame(AliRtcEngine.AliRtcAudioFrame frame) {
        // Callback for audio data to be published.
        Log.i(TAG, "onPublishAudioFrame");
        return true;
    }

    @Override
    public boolean onPlaybackAudioFrame(AliRtcEngine.AliRtcAudioFrame frame) {
        // Callback for playback data (after audio mixing).
        Log.i(TAG, "onPlaybackAudioFrame");
        return true;
    }

    @Override
    @Deprecated
    public boolean onMixedAllAudioFrame(AliRtcEngine.AliRtcAudioFrame frame) {
        // This method is deprecated.
        return true;
    }

    @Override
    public boolean onRemoteUserAudioFrame(String uid, AliRtcEngine.AliRtcAudioFrame frame) {
        // Callback for audio data from a specific remote user.
        Log.i(TAG, "onRemoteUserAudioFrame");
        return true;
    }
};

iOS

Implement the required methods in the AliRtcAudioFrameDelegate delegate.

// MARK: Audio data callbacks
extension ProcessAudioRawDataMainVC: AliRtcAudioFrameDelegate {
    func onCapturedAudioFrame(_ frame: AliRtcAudioFrame) -> Bool {
        let message = "onCaptureVideoSample: numofSamples: \(frame.numOfSamples), numofChannels: \(frame.numOfChannels), SampleRate: \(frame.samplesPerSec)"
        message.printLog()
        updateInfoLabel(captureInfoLabel, text: message)
        return true
    }
    
    func onProcessCapturedAudioFrame(_ frame: AliRtcAudioFrame) -> Bool {
        let message = "onProcessCapturedAudioFrame: numofSamples: \(frame.numOfSamples), numofChannels: \(frame.numOfChannels), SampleRate: \(frame.samplesPerSec)"
        message.printLog()
        updateInfoLabel(processInfoLabel, text: message)
        return true
    }
    
    func onPublishAudioFrame(_ frame: AliRtcAudioFrame) -> Bool {
        let message = "onPublishAudioFrame: numofSamples: \(frame.numOfSamples), numofChannels: \(frame.numOfChannels), SampleRate: \(frame.samplesPerSec)"
        message.printLog()
        updateInfoLabel(publishInfoLabel, text: message)
        return true
    }
    
    func onPlaybackAudioFrame(_ frame: AliRtcAudioFrame) -> Bool {
        let message = "onPlaybackAudioFrame: numofSamples: \(frame.numOfSamples), numofChannels: \(frame.numOfChannels), SampleRate: \(frame.samplesPerSec)"
        message.printLog()
        updateInfoLabel(prePlaybackInfoLabel, text: message)
        return true
    }
    
    func onRemoteUserAudioFrame(_ uid: String?, frame: AliRtcAudioFrame) -> Bool {
        let message = "onRemoteUserAudioFrame: uid: \(uid ?? "invalid"), numofSamples: \(frame.numOfSamples), numofChannels: \(frame.numOfChannels), SampleRate: \(frame.samplesPerSec)"
        message.printLog()
        updateInfoLabel(remoteUserInfoLabel, text: message)
        return true
    }
}

Harmony

private createAudioFrameObserver(): void {
  const rtcAudioFrameObserver :AliRtcEngineAudioFrameListener = {} as AliRtcEngineAudioFrameListener
    rtcAudioFrameObserver.onCapturedAudioFrame ((frame: AliRtcAudioFrame) => {
  // Callback for locally captured audio data.
  console.info('onCaptureAudioFrame');
  return true;
});

rtcAudioFrameObserver.onProcessCapturedAudioFrame((frame: AliRtcAudioFrame) => {
  // Callback for audio data after 3A processing.
  console.info('onProcessCaptureAudioFrame');
  return true;
});

rtcAudioFrameObserver.onPublishAudioFrame((frame: AliRtcAudioFrame) => {
  // Callback for audio data to be published.
  console.info('onPublishAudioFrame');
  return true;
});

rtcAudioFrameObserver.onPlaybackAudioFrame((frame: AliRtcAudioFrame) => {
  // Callback for playback data (after audio mixing).
  console.info('onPlaybackAudioFrame');
  return true;
});

rtcAudioFrameObserver.onRemoteUserAudioFrame((uid: string, frame: AliRtcAudioFrame) => {
  // Callback for audio data from a remote user.
  console.info(`onRemoteUserAudioFrame, uid: ${uid}`);
  return true;
})

// Register the audio frame observer.
if (this.rtcEngine) {
  this.rtcEngine.registerAudioFrameObserver(rtcAudioFrameObserver);
}
}

Mac

Implement the required methods of the AliRtcAudioFrameDelegate class.

- (BOOL)onCapturedAudioFrame:(AliRtcAudioFrame* _Nonnull)frame {
    if (_isSaveLocalAudio) {
        [self saveAudioWithUid:selfUid audioSample:frame audioSource:AliRtcAudioSourceCaptured];
    }
    if (_isModifyLocalAudio) {
        [self modifyAudio:(AliRtcAudioFrame *)frame];
    }
    return true;
}

- (BOOL)onProcessCapturedAudioFrame:(AliRtcAudioFrame* _Nonnull)frame {
    if (_isSaveLocalAudio) {
        [self saveAudioWithUid:selfUid audioSample:frame audioSource:AliRtcAudioSourceProcessCaptured];
    }
    if (_isModifyLocalAudio) {
        [self modifyAudio:(AliRtcAudioFrame *)frame];
    }
    return true;
}

- (BOOL)onPublishAudioFrame:(AliRtcAudioFrame* _Nonnull)frame {
    if (_isSaveLocalAudio) {
        [self saveAudioWithUid:selfUid audioSample:frame audioSource:AliRtcAudioSourcePub];
    }
    if (_isModifyLocalAudio) {
        [self modifyAudio:(AliRtcAudioFrame *)frame];
    }
    return true;
}

- (BOOL)onPlaybackAudioFrame:(AliRtcAudioFrame* _Nonnull)frame {
    if (_isSaveLocalAudio) {
        [self saveAudioWithUid:selfUid audioSample:frame audioSource:AliRtcAudioSourcePlayback];
    }
    if (_isModifyLocalAudio) {
        [self modifyAudio:(AliRtcAudioFrame *)frame];
    }
    return true;
}


- (BOOL)onRemoteUserAudioFrame:(NSString *_Nullable)uid frame:(AliRtcAudioFrame* _Nonnull)frame {
    BOOL canSave = NO;
    for (NSString *itemString in self.saveAudioUidArray) {
        if ([itemString isEqualToString:uid]) {
            canSave = YES;
            break;
        }
    }
    if (canSave) {
        [self saveAudioWithUid:uid audioSample:frame audioSource:(AliRtcAudioSource)AliRtcAudioSourceRemoteUser];
    }
    if (_isModifyLocalAudio) {
        [self modifyAudio:(AliRtcAudioFrame *)frame];
    }

    return true;
}

Windows

Implement the required methods of the IAudioFrameObserver class.

bool CTutorialDlg::OnPublishAudioFrame(AliEngineAudioRawData audioRawData)
{
	if (NULL == audioRawData.dataPtr) return false;

	// save audio
	if (cshell_dlg.mbSavePcmAudio) {
		tag_AudioDataSample *p = new tag_AudioDataSample(AliEngineAudioSourcePub, &audioRawData);
		m_MutexSaveaudio.Lock();
		m_listSavePubAudio.push_back(p);
		m_MutexSaveaudio.Unlock();
	}

	// modify audio
	if (mbModifyAudio) {
		modifyDataAudioFrame(audioRawData);
	}
	return true;
}

bool CTutorialDlg::OnCapturedAudioFrame(AliEngineAudioRawData audioRawData) {
	if (audioRawData.dataPtr == NULL) return false;

	// save audio
	if (cshell_dlg.mbSavePcmAudio) {
		tag_AudioDataSample *p = new tag_AudioDataSample(AliEngineAudioSourceCaptured, &audioRawData);
		m_MutexSaveaudio.Lock();
		m_listSaveRawAudio.push_back(p);
		m_MutexSaveaudio.Unlock();
	}
	// modify audio
	if (mbModifyAudio) {
		modifyDataAudioFrame(audioRawData);
	}
 	return true;
}

bool CTutorialDlg::OnProcessCapturedAudioFrame(AliEngineAudioRawData audioRawData) {
	if (NULL == audioRawData.dataPtr) return false;

	// save audio
	if (cshell_dlg.mbSavePcmAudio) {
		tag_AudioDataSample *p = new tag_AudioDataSample(AliEngineAudioSourceProcessCaptured, &audioRawData);
		m_MutexSaveaudio.Lock();
		m_listSaveProcessedRawAudio.push_back(p);
		m_MutexSaveaudio.Unlock();
	}
	// modify audio
	if (mbModifyAudio) {
		modifyDataAudioFrame(audioRawData);
	}
	return true;
}

bool CTutorialDlg::OnPlaybackAudioFrame(AliEngineAudioRawData audioRawData) {
	if (NULL == audioRawData.dataPtr) return false;

	// save audio
	if (cshell_dlg.mbSavePcmAudio) {
		tag_AudioDataSample *p = new tag_AudioDataSample(AliEngineAudioSourcePlayback, &audioRawData);
		m_MutexSaveaudio.Lock();
		m_listSaveSubAudio.push_back(p);
		m_MutexSaveaudio.Unlock();
	}

	// modify audio
	if (mbModifyAudio) {
		modifyDataAudioFrame(audioRawData);
	}
	return true;
}

bool CTutorialDlg::OnRemoteUserAudioFrame(const char *uid, AliEngineAudioRawData audioRawData) {
	PartCfgDlg *partView = mRemotePartDlgs[uid];
	if (NULL == audioRawData.dataPtr) return false;

	// save audio
	if (nullptr != partView  && partView->mbSaveAudioData) {
		tag_AudioDataSample *p = new tag_AudioDataSample(AliEngineAudioSourceRemoteUser, &audioRawData);
		m_MutexSaveUserAudio.Lock();
		m_multimapSaveUserAudio.insert(std::pair<std::string, tag_AudioDataSample*>(uid, p));
		m_MutexSaveUserAudio.Unlock();
	}

	// modify audio
	if (mbModifyAudio) {
		modifyDataAudioFrame(audioRawData);
	}

	return true;
}

3. Enable raw audio data callbacks

  1. Call the registerAudioFrameObserver method to register the audio frame observer.

  2. Call the enableAudioFrameObserver method to enable the callback for a specific audio source and configure the data format. To enable multiple callbacks, call this method for each source.

    • enable: A boolean value. Set this to true to enable the callback or false to disable it.

    • audioSource: The type of audio source, which corresponds to one of the callbacks.

    • config: The configuration for the callback data. If set to null, the default configuration is used: a sample rate of 48,000 Hz, mono channel, and read-only mode.

    Note

    • Each call to enableAudioFrameObserver corresponds to a specific audio data callback. Call this method multiple times to observe different audio sources.

    • You can call enableAudioFrameObserver both before and after joining a channel.

  3. Receive and process the audio data within the corresponding callback methods.

Android

// Register the observer for raw audio frames.
mAliRtcEngine.registerAudioFrameObserver(rtcAudioFrameObserver);

// Enable the desired callbacks based on your use case.
mAliRtcEngine.enableAudioFrameObserver(isChecked, AliRtcEngine.AliRtcAudioSource.AliRtcAudioSourceCaptured, config);
mAliRtcEngine.enableAudioFrameObserver(isChecked, AliRtcEngine.AliRtcAudioSource.AliRtcAudioSourceProcessCaptured, config);
mAliRtcEngine.enableAudioFrameObserver(isChecked, AliRtcEngine.AliRtcAudioSource.AliRtcAudioSourcePub, config);
mAliRtcEngine.enableAudioFrameObserver(isChecked, AliRtcEngine.AliRtcAudioSource.AliRtcAudioSourcePlayback, config);
mAliRtcEngine.enableAudioFrameObserver(isChecked, AliRtcEngine.AliRtcAudioSource.AliRtcAudioSourceRemoteUser, config);

iOS

// Register the audio frame observer.
self.rtcEngine?.registerAudioFrameObserver(self)

// Enable the desired callbacks based on your use case. In this example, the settings are based on a configuration object.
for (source, config) in configs {
    if config.enable, let observerConfig = config.observerConfig {
        // Enable the audio frame observer.
        self.rtcEngine?.enableAudioFrameObserver(config.enable, audioSource: source, config: observerConfig)
        isEnableAudioRawFrame = true
    }
}

Harmony

// Register the audio frame observer.
this.rtcEngine.registerAudioFrameObserver(rtcAudioFrameObserver);

// Enable the desired callbacks based on your use case. In this example, the settings are based on a configuration object.
const config: AliRtcAudioFrameObserverConfig = new AliRtcAudioFrameObserverConfig();
config.sampleRate = AliRtcAudioSampleRate.AliRtcAudioSampleRate_48000;
config.channels = AliRtcAudioNumChannel.AliRtcMonoAudio;

this.rtcEngine.enableAudioFrameObserver(isChecked, AliRtcAudioSource.AliRtcAudioSourceCaptured, config);

Mac

[_engine registerAudioFrameObserver:self];

AliRtcAudioFrameObserverConfig* config = [[AliRtcAudioFrameObserverConfig alloc] init];

config.sampleRate = AliRtcAudioSampleRate_48000;
config.channels = audioNumChannel;
config.mode = AliRtcAudioFrameObserverOperationModeReadWrite;

[_engine enableAudioFrameObserver:TRUE audioSource:AliRtcAudioSourcePub config:config];
[_engine enableAudioFrameObserver:TRUE audioSource:AliRtcAudioSourcePlayback config:config];
[_engine enableAudioFrameObserver:TRUE audioSource:AliRtcAudioSourceCaptured config:config];
[_engine enableAudioFrameObserver:TRUE audioSource:AliRtcAudioSourceProcessCaptured config:config];
[_engine enableAudioFrameObserver:TRUE audioSource:AliRtcAudioSourceRemoteUser];

Windows

// Register the observer for raw audio frames.
mAliRtcEngine->RegisterAudioFrameObserver(mAudioFrameObserver);

AliEngineAudioFrameObserverConfig config;

config.sampleRate = AliEngineAudioSampleRate_48000;
config.channels   = AliEngineStereoAudio;

// Enable the desired callbacks based on your use case.
mAliRtcEngine->EnableAudioFrameObserver(true, AliEngineAudioSourceCaptured, config);
mAliRtcEngine->EnableAudioFrameObserver(true, AliEngineAudioSourceProcessCaptured, config);
mAliRtcEngine->EnableAudioFrameObserver(true, AliEngineAudioSourcePub, config);
mAliRtcEngine->EnableAudioFrameObserver(true, AliEngineAudioSourcePlayback, config);
mAliRtcEngine->EnableAudioFrameObserver(true, AliEngineAudioSourceRemoteUser, config);

4. Stop the audio data callbacks

To stop receiving audio data callbacks, either call registerAudioFrameObserver(null) to unregister the observer, or call enableAudioFrameObserver with the enable parameter set to false to disable a specific audio source.

Android

// Stop the audio data callback for a specific audio source.
mAliRtcEngine.enableAudioFrameObserver(false, AliRtcEngine.AliRtcAudioSource.AliRtcAudioSourceCaptured, config);

// Unregister the observer.
mAliRtcEngine.registerAudioFrameObserver(null);

iOS

// Stop the audio data callback for a specific audio source.
self.rtcEngine?.enableAudioFrameObserver(false, audioSource: source, config: observerConfig)

// Unregister the observer.
self.rtcEngine?.registerAudioFrameObserver(nil)

Harmony

// Stop the audio data callback for a specific audio source.
this.rtcEngine.enableAudioFrameObserver(false, AliRtcAudioSource.AliRtcAudioSourceCaptured, config);

// Unregister the observer.
self.rtcEngine?.registerAudioFrameObserver(nil)

Notes

You must call the enableAudioFrameObserver method to enable each audio source callback. Otherwise, the callback is not triggered.

Previous：Custom audio playbackNext：Video management