通过阅读本文,您可以快速了解 iOS 远程双录的相关功能及调用方法。
远程双录功能依赖 MPIDRSSDK 和音视频通话组件( Mobile Real-Time Communication,简称 MRTC),MTRC 中的相关 API 请参见 使用 iOS SDK。
iOS 接入
创建项目工程
使用 xcode 创建一个新的项目。

环境配置
SDK 环境依赖
MPIDRSSDK 为动态库,支持 iOS 9.0 及以上系统。
进入 Targets > Build Phases,添加 MPIDRSSDK 及 ARTVC 依赖库,如下图所示。
参数配置
进入 Targets > Build Phases,添加 MPIDRSSDK、ARTVC 动态库,如下图所示。
添加 MPIDRSSDK.bundle 资源,如下图所示。
将 Other Linker Flags 设置为 -ObjC,如下图所示。
支持 HTTP 请求,即在
App Transport Security Settings
字段下加入Allow Arbitrary Loads
,并配置为 YES,如下图所示。允许使用相机、麦克风权限。
初始化 MPIDRSSDK
/// 初始化
/// @param recordType 录制类型
/// @param userId userId
/// @param appId appid
/// @param packageName packageName
/// @param ak ak
/// @param sk sk
/// @param success sdk返回
/// @param failure 失败返回
[MPIDRSSDK initWithRecordType:IDRSRecordRemote userId:@"xxx" appId:AppId packageName:PackageName AK:Ak SK:Sk success:^(id _Nonnull responseObject) {
self->_idrsSDK = responseObject;
self->_idrsSDK.nui_tts_delegate = self;
} failure:^(NSError * _Nonnull error) {
NSLog(@"----%@",error);
}];
IDRSNUITTSDelegate
/// player播放完成
- (void)onPlayerDidFinish;
/// tts合成状态
/// @param event 状态
/// @param taskid 任务id
/// @param code code
- (void)onNuiTtsEventCallback:(ISDRTtsEvent)event taskId:(NSString *)taskid code:(int)code;
/// tts合成数据
/// @param info 文本信息
/// @param info_len 下角标(不是很准、推荐使用info.length)
/// @param buffer 音频数据流
/// @param len 数据流长度
/// @param task_id 任务id
- (void)onNuiTtsUserdataCallback:(NSString *)info infoLen:(int)info_len buffer:(char*)buffer len:(int)len taskId:(NSString *)task_id;
/// nui识别结果
/// @param result 激活词识别的结果
- (void)onNuiKwsResultCallback:(NSString *)result;
面部检测
/// 设置人脸检测的参数
/// @param value 参数的值,详情见idrs_face_param_type枚举
/// @param key idrs_face_param_type枚举
- (void)setFaceDetectParameter:(float)value forKey:(idrs_face_param_type)key;
/// 检测人脸特征值
/// @param faceDetectParam IDRSFaceDetectParam
- (NSArray<FaceDetectionOutput *> *)detectFace:(nonnull IDRSFaceDetectParam *)faceDetectParam;
/// 人照比对
/// @param faceFea1 脸1的特征值
/// @param faceFea2 脸2的特征值
- (float)faceRecognitionSimilarity:(nonnull NSArray<NSNumber*>*)faceFea1
feature2:(nonnull NSArray<NSNumber*>*)faceFea2;
/// 用图片引擎做人脸追踪
/// @param faceDetectParam IDRSFaceDetectParam
/// @param block 结果返回
- (void)faceTrackFromImage:(nonnull IDRSFaceDetectParam *)faceDetectParam
faceDetectionCallback:(void (^)(NSError *error, NSArray<FaceDetectionOutput*> *faces))block;// 非主线程回调;
/// 用视频引擎中人脸追踪
/// @param faceDetectParam IDRSFaceDetectParam
/// @param faceDetectionCallbackBlock 结果返回
- (void)faceTrackFromVideo:(nonnull IDRSFaceDetectParam *)faceDetectParam
faceDetectionCallback:(void (^)(NSError *error, NSArray<FaceDetectionOutput*> *faces))faceDetectionCallbackBlock;
/// 用视频引擎中RTC_远端窗口--人脸追踪
/// @param faceDetectParam IDRSFaceDetectParam
/// @param faceDetectionCallbackBlock 结果返回
- (void)faceTrackFromRemoteVideo:(nonnull IDRSFaceDetectParam *)faceDetectParam
faceDetectionCallback:(void (^)(NSError *error, NSArray<FaceDetectionOutput*> *faces))faceDetectionCallbackBlock;
手势检测
/// 设置手势检测参数。需要在调用detectHandGesture之前调用,才能起作用
/// @param handDetectConfig IDRSHandDetectionConfig
-(void)setHandDetectConfig:(nonnull IDRSHandDetectionConfig *)handDetectConfig;
/// 检测动态手势,手持手机签字
/// @param handParam IDRSHandDetectParam
- (NSArray<HandDetectionOutput *> *)detectHandGesture:(nonnull IDRSHandDetectParam *)handParam;
/// 检测静态手势
/// @param handParam IDRSHandDetectParam
- (NSArray<HandDetectionOutput *> *)detectHandStaticGesture:(nonnull IDRSHandDetectParam *)handParam;
签名类型
// 签名类型检测
/// @param cameraFrame 相机流
/// @param roikey ROI数组
-(NSArray <IDRSSignConfidenceCheck *>*)checkSignClassifyWithCameraBuffer:(CVPixelBufferRef)cameraFrame AndROI:(NSArray<NSNumber*>*)roikey;
/// 签名类型检测
/// @param image 所检测的image
/// @param roikey ROI数组
-(NSArray <IDRSSignConfidenceCheck *>*)checkSignClassifyWithImage:(UIImage*)image AndROI:(NSArray<NSNumber*>*)roikey;
身份证
/// 检测身份证OCR
/// @param idCardParam IDRSIDCardDetectParam
/// @param roiKey ROI数组
/// @param rotate 角度
/// @param frontCamera 是否为前置摄像头:YES:前置,NO:后置
/// @param frontIDCard 是否检测身份证带人面:YES:带人面,NO:国徽面
- (IDCardDetectionOutput *)detectIDCard:(nonnull IDRSIDCardDetectParam *)idCardParam
roiKey:(nonnull NSArray<NSNumber*>*)roiKey
rotate:(NSNumber*)rotate
isFrontCamera:(BOOL)frontCamera
isDetectFrontIDCard:(BOOL)frontIDCard;
NUI 识别(关键词、激活词)
/// 开启激活词检测
- (void)startDialog;
/// 结束激活词检测
- (void)stopDialog;
//如果检测非本地麦克风收集的声音,需使用:
/**
激活词外部输入数据
@param voiceFrame 音频数据(需要:pcm、单声道、16k采样率)
*/
- (void)feedAudioFrame:(NSData*)voiceFrame;
TTS
/// 开始TTS合成
/// @param text 需要合成语音的文本,如:@“这是测试文本”
- (void)startTTSWithText:(NSString *)text;
/// 结束TTS合成
- (void)stopTTS;
/// 暂停当前播放(与resume相反)
- (void)pauseTTS;
/// 重启当前播放(与pause相反)
- (void)resumeTTS;
/// 设置参数,设置成功后下次合成将采用新的参数
/// @param param param: extend_font_name:语音模型 value: (xiaoyun/xiaogang)
/// param: speed_level:语音速度 value:(0-2],默认1.0
/// param: volume: 音量 value: [0,2.6],推荐1.5以下
/// param: pitch_level:音调,value:[-500,500]
- (void)setTTSParam:(NSString *)param
value:(NSString *)value;
/// 获取TTS参数
/// @param param font_name、mode_type、speed_level、volume
-(NSString *)getTTSParam:(NSString *)param;
Meta 文件相关
/// 初始化meta文件及result文件计时器
-(void)startInitTime;
/// 保存Meta文件
/// @param fileName 文件名,需带后缀:xxx.meta
/// @param filePath 文件路径,如果为nil或@"",则存在默认路径
-(NSString*)saveMetaWithfileName:(NSString*)fileName andfilePath:(NSString*)filePath;
/// @param sequence 可理解为:章节
- (void)startSegment:(NSString *)sequence;
/// @param idCard 给某个章节添加身份证号
- (void)addSegment:(NSString *)sequence andIDCard:(NSString*)idCard;
- (void)endSegment:(NSString *)sequence;
/// @param actionName 开始某个章节的actionName检测项
- (void)startActionInSegment:(NSString *)sequence
actionName:(NSString *)actionName;
- (void)endActionInSegment:(NSString *)sequence
actionName:(NSString *)actionName;
/// @param word 识别激活词
- (void)startAddWord:(NSString *)word;
- (void)endAddWord:(NSString *)word;
/// @param label 人物名称
/// @param imageBase64 人脸照片的base64
- (void)addFace:(NSString *)label
image:(NSString *)imageBase64;
/// @param holderIdCardNumber 身份证号
/// @param title 文档标题
- (void)addPolicy:(NSString *)holderIdCardNumber
title:(NSString *)title;
/// @param detection 多端检测统一码
-(void)addDetection:(NSString*)detection;
result 文件相关
/// 开始某章某节的检测项
/// @param sequence 第几章
/// @param action 第几小节(用于区分同一章节多个相同的检测)
/// @param actionName 检测项名称
- (void)resultStartActionInSegment:(NSString *)sequence
action:(NSString *)action
actionName:(NSString *)actionName;
- (void)resultEndActionInSegment:(NSString *)sequence
action:(NSString *)action;
/// 给某一章某一节添加结果
/// @param sequence 第几章
/// @param action 第几小节
/// @param result 检测结果
- (void)resultAddActionInSegment:(NSString *)sequence
action:(NSString *)action
result:(NSString *)result;
/// 给某一章某一节添加照片(base64)
/// @param sequence 第几章
/// @param action 第几小节
/// @param image 图片(暂时只获取身份证图片)
- (void)resultAddActionInSegment:(NSString *)sequence
action:(NSString *)action
image:(NSString *)image;
/// @param md5String 录制视频的md5加密
- (void)resultAddVideoMD5:(NSString *)md5String;
/// @param detection 统一标识码
- (void)resultAddDetection:(NSString *)detection;
/// 获取resultJson
- (NSString *)resultGetMetaJson;
/// 保存result文件
/// @param fileName 文件名,需带后缀:xxx.result.meta
/// @param filePath 文件路径,如果为nil或@"",则存在默认路径
- (NSString *)resultSaveToFile:(NSString *)fileName andPath:(NSString*)filePath;
tools
/// 获取SDK版本号
+ (NSString*)getVersion;
/// 将相机中获取的sampleBuffer保存到相册
/// @param sampleBuffer CMSampleBufferRef
- (void)saveSampleBufferFromCamera: (CMSampleBufferRef)sampleBuffer;
/// 将rtc视频流转成图片
/// @param imageBuffer CVPixelBufferRef、CVPixelBufferRef
- (UIImage *) getImageFromRPVideo: (CVImageBufferRef)imageBuffer;
释放
/// 释放资源
- (void)releaseResources;
初始化 RTC 实例
[MPIDRSSDK initRTCWithUserId:self.uid appId:AppId success:^(id _Nonnull responseObject) {
self.artvcEgnine = responseObject;
self.artvcEgnine.delegate = self;
self.artvcEgnine.videoProfileType = ARTVCVideoProfileType_640x360_15Fps;
// 发布
ARTVCPublishConfig *publishConfig = [[ARTVCPublishConfig alloc] init];
publishConfig.videoProfile = self.artvcEgnine.videoProfileType;
publishConfig.audioEnable = YES;
publishConfig.videoEnable = YES;
self.artvcEgnine.autoPublishConfig = publishConfig;
self.artvcEgnine.autoPublish = YES;
// 订阅
ARTVCSubscribeOptions *subscribeOptions = [[ARTVCSubscribeOptions alloc] init];
subscribeOptions.receiveAudio = YES;
subscribeOptions.receiveVideo = YES;
self.artvcEgnine.autoSubscribe = subscribeOptions;
self.artvcEgnine.autoSubscribe = YES;
//订阅本地视频流和音频流
self.artvcEgnine.enableAudioBufferOutput = YES;
self.artvcEgnine.enableCameraRawSampleOutput = YES;
self.artvcEgnine.expectedAudioPlayMode = ARTVCAudioPlayModeSpeaker;
self.artvcEgnine.degradationPreference = ARTVCDegradationPreferenceMAINTAIN_FRAMERATE;
[self.artvcEgnine startCameraPreviewUsingBackCamera:NO];
if (self.roomId && self.rtoken) {
self.isOther = false;
[IDRSSDK joinRoom:self.roomId token:self.rtoken];
}else {
self.isOther = true;
[IDRSSDK createRoom];
}
} failure:^(NSError * _Nonnull error) {
}];
视频相关检测
// 此方法为ARTVC 实例代理方法,详见:ARTVCEngineDelegate
- (void)didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer {
__weak __typeof(self) weakSelf = self;
dispatch_queue_t testqueue = dispatch_queue_create("subVideo", NULL);
dispatch_sync(testqueue, ^{
CVPixelBufferRef newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) ;
if (_isWrite) {//签名检测;
UIImage * image = [self.idrs getImageFromRPVideo:newBuffer];
NSArray<NSNumber*> *kXMediaOptionsROIKey = @[@(0.2),@(0.2),@(0.6),@(0.6)];
NSArray<IDRSSignConfidenceCheck *>*sings = [self.idrs checkSignClassifyWithImage:image AndROI:kXMediaOptionsROIKey];
dispatch_async(dispatch_get_main_queue(), ^{
NSString*string = @"";
float max = 0.0;
for (IDRSSignConfidenceCheck* Confidence in sings) {
if (Confidence.confidence > max) {
string = Confidence.label;
max = Confidence.confidence;
}
}
if ([string isEqual:@"hand"] && max > 0.95) {
self.resultLabel.text = [NSString stringWithFormat:@"手写体"];
[self didsendMessage:@"检测到手写体" isOn:true type:0];
}
});
}
if(_isFaceCheck){
@autoreleasepool {
//用图片检测
IDRSFaceDetectParam *detectParam = [[IDRSFaceDetectParam alloc]init];
detectParam.dataType = IDRSFaceDetectInputTypePixelBuffer;
detectParam.buffer = newBuffer;
detectParam.inputAngle = 0;
detectParam.outputAngle = 0;
detectParam.faceNetType = 0;
detectParam.supportFaceRecognition = false;
detectParam.supportFaceLiveness = false;
[_idrs faceTrackFromVideo:detectParam faceDetectionCallback:^(NSError *error, NSArray<FaceDetectionOutput *> *faces) {
//脸的标识框:
dispatch_async(dispatch_get_main_queue(), ^{
self.drawView.faceDetectView.detectResult = faces;
});
}];
}
}else{
dispatch_async(dispatch_get_main_queue(), ^{
self.drawView.faceDetectView.detectResult = nil;
});
}//人脸框结束;
if (weakSelf.isOCR_Front || weakSelf.isOCR_Back) {
@autoreleasepool {
BOOL isForont = weakSelf.isOCR_Front? true:false;
// 2. ocr身份证。
IDRSIDCardDetectParam *idCardParam = [[IDRSIDCardDetectParam alloc]init];
idCardParam.dataType = IDRSIDCardInputTypePixelBuffer;
idCardParam.buffer = newBuffer;
NSArray<NSNumber*> *kXMediaOptionsROIKey = @[@(0.2),@(0.2),@(0.6),@(0.6)];
IDCardDetectionOutput *ocrResult = [_idrs detectIDCard:idCardParam roiKey:kXMediaOptionsROIKey rotate:@(90) isFrontCamera:NO isDetectFrontIDCard:isForont];
if (ocrResult!=nil) {
NSString *result = @"";
if (ocrResult.num.length > 0) {
result = ocrResult.num;
}else if (ocrResult.date.length > 0){
result = ocrResult.date;
}
// 采集到了身份证信息
dispatch_async(dispatch_get_main_queue(), ^{
// 显示身份证信息
self.resultLabel.text = result;
[self didsendMessage:result isOn:true type:0];
});
}
}
}
// 3. 手势检测
if (weakSelf.isHandDet) {
@autoreleasepool {
IDRSHandDetectParam *handParam = [[IDRSHandDetectParam alloc]init];
handParam.dataType = IDRSHandInputTypeRGBA;
handParam.buffer = newBuffer;
handParam.outAngle = 0;
NSArray<HandDetectionOutput *> *handResults = [_idrs detectHandGesture:handParam];
if(handResults.count > 0) {
// 判断签字
if (handResults[0].phone_touched_score>0) {
//手写框
dispatch_async(dispatch_get_main_queue(), ^{
weakSelf.drawView.handDetectView.detectResult = handResults;
if(handResults[0].hand_phone_action == 1 || handResults[0].hand_phone_action == 2) {
self.resultLabel.text = @"签字成功";
[self didsendMessage:@"签字成功" isOn:true type:0];
}
});
}
}else{
dispatch_async(dispatch_get_main_queue(), ^{
weakSelf.drawView.handDetectView.detectResult = nil;
});
}
}
}
//静态手势检测
if(self.isStaticHandDet) {
@autoreleasepool {
IDRSHandDetectParam *handParam = [[IDRSHandDetectParam alloc]init];
handParam.dataType = IDRSHandInputTypeRGBA;
handParam.buffer = newBuffer;
handParam.outAngle = 0;
NSArray<HandDetectionOutput *> *handResults = [_idrs detectHandStaticGesture:handParam];
dispatch_async(dispatch_get_main_queue(), ^{
if (handResults.count > 0) {
weakSelf.drawView.handDetectView.detectResult = handResults;
HandDetectionOutput *handResult = handResults[0];
if (handResult.hand_action_type == 0 && handResult.hand_static_action > 0) {
NSString *result = [self handStaticActionTypeToText:handResult.hand_static_action];
self.resultLabel.text = result;
[self didsendMessage:result isOn:true type:0];
}
}else{
dispatch_async(dispatch_get_main_queue(), ^{
weakSelf.drawView.handDetectView.detectResult = nil;
});
}
});
}
}
if (!weakSelf.isHandDet && !weakSelf.isStaticHandDet) {
dispatch_async(dispatch_get_main_queue(), ^{
weakSelf.drawView.handDetectView.detectResult = nil;
});
}
});
}
音频相关检测
// 此方法为ARTVC 实例代理方法,详见:ARTVCEngineDelegate
- (void)didOutputAudioBuffer:(ARTVCAudioData*)audioData {
if (_isNui) {
if (audioData.audioBufferList->mBuffers[0].mData != NULL && audioData.audioBufferList->mBuffers[0].mDataByteSize > 0) {
// NSData *srcData = [NSData dataWithBytes:audioData.audioBufferList->mBuffers[0].mData length:audioData.audioBufferList->mBuffers[0].mDataByteSize];
//重采样为16K
pcm_frame_t pcmModelInput;
pcmModelInput.len = audioData.audioBufferList->mBuffers[0].mDataByteSize;
pcmModelInput.buf = (uint8_t*)audioData.audioBufferList->mBuffers[0].mData;
pcmModelInput.sample_rate = audioData.sampleRate;
pcm_frame_t pcmModelOutput;
pcm_resample_16k(&pcmModelInput, &pcmModelOutput);
NSData *srcData = [NSData dataWithBytes:pcmModelOutput.buf length:pcmModelOutput.len];
[self.idrs feedAudioFrame:srcData];
}
}
}
录制功能
说明
成功初始化 MPIDRSSDK 和 ARTVC 实例以后调用才生效。
设置水印并开启远程录制
[MPIDRSSDK openRemoteRecordWithRoomId:self.roomId
waterMarkId:self.watermarkId];
关闭远程录制
+ (void)stopRemoteRecord:(NSString *)recordId;
录制回调
- (void)didReceiveCustomSignalingResponse:(NSDictionary *)dictionary{
id opcmdObject = [dictionary objectForKey:@"opcmd"];
if ([opcmdObject isKindOfClass:[NSNumber class]]) {
int opcmd = [opcmdObject intValue];
switch (opcmd) {
case 1032: {
NSLog(@"开始录制");
self.startTime = [NSDate date];
self.recordId = [dictionary objectForKey:@"recordId"];
}
break;
case 1034: {
NSLog(@"结束录制");
self.duration = [[NSDate date] timeIntervalSince1970] - [self.startTime timeIntervalSince1970];
}
break;
default:
break;
} }
}
上传录制产物
self.duration = [[NSDate date] timeIntervalSince1970] - [self.startTime timeIntervalSince1970];
IDRSUploadManagerParam *param = [[IDRSUploadManagerParam alloc] init];
param.duration = self.duration;
param.appId = AppId;
param.ak = Ak;
param.sk = Sk;
param.type = IDRSRecordRemote;
param.recordAt = self.startTime;
param.roomId = self.roomId;
[IDRSUploadManager uploadFileWithParam:param success:^(id _Nonnull responseObject) {
[self showToastWith:responseObject duration:3];
} failure:^(NSError * _Nonnull error, IDRSUploadManagerParam * _Nonnull upLoadParam) {
//
if (upLoadParam) {
[self showToastWith:@"upload error" duration:3];
}
}];