阿里云首页 智能双录质检 相关技术圈

远程双录

通过阅读本文，您可以快速了解 iOS 远程双录的相关功能及调用方法。

远程双录功能依赖 MPIDRSSDK 和音视频通话组件（ Mobile Real-Time Communication，简称 MRTC）,MTRC 中的相关 API 请参见 使用 iOS SDK。

iOS 接入

创建项目工程

使用 xcode 创建一个新的项目。

1

环境配置

  1. SDK 环境依赖

    MPIDRSSDK 为动态库，支持 iOS 9.0 及以上系统。

    进入 Targets > Build Phases，添加 MPIDRSSDK 及 ARTVC 依赖库，如下图所示。

    22

  2. 参数配置

    1. 进入 Targets > Build Phases，添加 MPIDRSSDK、ARTVC 动态库，如下图所示。

      33

    2. 添加 MPIDRSSDK.bundle 资源，如下图所示。

      44

    3. 将 Other Linker Flags 设置为 -ObjC，如下图所示。

      4

    4. 支持 HTTP 请求，即在 App Transport Security Settings 字段下加入 Allow Arbitrary Loads，并配置为 YES，如下图所示。

      5

    5. 允许使用相机、麦克风权限。

      223

初始化 MPIDRSSDK

/// 初始化
/// @param recordType 录制类型
/// @param userId userId
/// @param appId appid
/// @param packageName packageName
/// @param ak ak
/// @param sk sk
/// @param success sdk返回
/// @param failure 失败返回
[MPIDRSSDK initWithRecordType:IDRSRecordRemote userId:@"xxx" appId:AppId packageName:PackageName AK:Ak SK:Sk success:^(id  _Nonnull responseObject) {
    self->_idrsSDK = responseObject;
    self->_idrsSDK.nui_tts_delegate = self;
} failure:^(NSError * _Nonnull error) {
    NSLog(@"----%@",error);
}];

IDRSNUITTSDelegate

/// player播放完成
- (void)onPlayerDidFinish;

/// tts合成状态
/// @param event 状态
/// @param taskid 任务id
/// @param code code
- (void)onNuiTtsEventCallback:(ISDRTtsEvent)event taskId:(NSString *)taskid code:(int)code;

/// tts合成数据
/// @param info 文本信息
/// @param info_len 下角标（不是很准、推荐使用info.length）
/// @param buffer 音频数据流
/// @param len 数据流长度
/// @param task_id 任务id
- (void)onNuiTtsUserdataCallback:(NSString *)info infoLen:(int)info_len buffer:(char*)buffer len:(int)len taskId:(NSString *)task_id;

/// nui识别结果
/// @param result 激活词识别的结果
- (void)onNuiKwsResultCallback:(NSString *)result;

面部检测

/// 设置人脸检测的参数
/// @param value  参数的值,详情见idrs_face_param_type枚举
/// @param key idrs_face_param_type枚举
- (void)setFaceDetectParameter:(float)value forKey:(idrs_face_param_type)key;

/// 检测人脸特征值
/// @param faceDetectParam IDRSFaceDetectParam
- (NSArray<FaceDetectionOutput *> *)detectFace:(nonnull IDRSFaceDetectParam *)faceDetectParam;

/// 人照比对
/// @param faceFea1 脸1的特征值
/// @param faceFea2 脸2的特征值
- (float)faceRecognitionSimilarity:(nonnull NSArray<NSNumber*>*)faceFea1
        feature2:(nonnull NSArray<NSNumber*>*)faceFea2;

/// 用图片引擎做人脸追踪
/// @param faceDetectParam IDRSFaceDetectParam
/// @param block 结果返回
- (void)faceTrackFromImage:(nonnull IDRSFaceDetectParam *)faceDetectParam
        faceDetectionCallback:(void (^)(NSError *error, NSArray<FaceDetectionOutput*> *faces))block;// 非主线程回调;

/// 用视频引擎中人脸追踪
/// @param faceDetectParam IDRSFaceDetectParam
/// @param faceDetectionCallbackBlock 结果返回
- (void)faceTrackFromVideo:(nonnull IDRSFaceDetectParam *)faceDetectParam
        faceDetectionCallback:(void (^)(NSError *error, NSArray<FaceDetectionOutput*> *faces))faceDetectionCallbackBlock;

/// 用视频引擎中RTC_远端窗口--人脸追踪
/// @param faceDetectParam IDRSFaceDetectParam
/// @param faceDetectionCallbackBlock 结果返回
- (void)faceTrackFromRemoteVideo:(nonnull IDRSFaceDetectParam *)faceDetectParam
        faceDetectionCallback:(void (^)(NSError *error, NSArray<FaceDetectionOutput*> *faces))faceDetectionCallbackBlock;

手势检测

/// 设置手势检测参数。需要在调用detectHandGesture之前调用，才能起作用
/// @param handDetectConfig IDRSHandDetectionConfig
-(void)setHandDetectConfig:(nonnull IDRSHandDetectionConfig *)handDetectConfig;

/// 检测动态手势，手持手机签字
/// @param handParam IDRSHandDetectParam
- (NSArray<HandDetectionOutput *> *)detectHandGesture:(nonnull IDRSHandDetectParam *)handParam;

/// 检测静态手势
/// @param handParam IDRSHandDetectParam
- (NSArray<HandDetectionOutput *> *)detectHandStaticGesture:(nonnull IDRSHandDetectParam *)handParam;

签名类型

// 签名类型检测
/// @param cameraFrame 相机流
/// @param roikey ROI数组
-(NSArray <IDRSSignConfidenceCheck *>*)checkSignClassifyWithCameraBuffer:(CVPixelBufferRef)cameraFrame AndROI:(NSArray<NSNumber*>*)roikey;

/// 签名类型检测
/// @param image 所检测的image
/// @param roikey ROI数组
-(NSArray <IDRSSignConfidenceCheck *>*)checkSignClassifyWithImage:(UIImage*)image AndROI:(NSArray<NSNumber*>*)roikey;

身份证

/// 检测身份证OCR
/// @param idCardParam IDRSIDCardDetectParam
/// @param roiKey ROI数组
/// @param rotate 角度
/// @param frontCamera 是否为前置摄像头：YES：前置，NO：后置
/// @param frontIDCard 是否检测身份证带人面：YES：带人面，NO：国徽面
- (IDCardDetectionOutput *)detectIDCard:(nonnull IDRSIDCardDetectParam *)idCardParam
        roiKey:(nonnull NSArray<NSNumber*>*)roiKey
        rotate:(NSNumber*)rotate
        isFrontCamera:(BOOL)frontCamera
        isDetectFrontIDCard:(BOOL)frontIDCard;

NUI 识别（关键词、激活词）

/// 开启激活词检测
- (void)startDialog;

/// 结束激活词检测
- (void)stopDialog;
//如果检测非本地麦克风收集的声音，需使用：
/**
   激活词外部输入数据
   @param voiceFrame 音频数据(需要：pcm、单声道、16k采样率)
 */
- (void)feedAudioFrame:(NSData*)voiceFrame;

TTS

/// 开始TTS合成
/// @param text 需要合成语音的文本，如：@“这是测试文本”
- (void)startTTSWithText:(NSString *)text;

/// 结束TTS合成
- (void)stopTTS;

/// 暂停当前播放（与resume相反）
- (void)pauseTTS;

/// 重启当前播放（与pause相反）
- (void)resumeTTS;

/// 设置参数，设置成功后下次合成将采用新的参数
/// @param param param:  extend_font_name：语音模型  value: (xiaoyun/xiaogang)
/// param:  speed_level：语音速度 value：(0-2],默认1.0
/// param:  volume: 音量 value: [0,2.6],推荐1.5以下
/// param:  pitch_level：音调，value：[-500,500]
- (void)setTTSParam:(NSString *)param
        value:(NSString *)value;

/// 获取TTS参数
/// @param param font_name、mode_type、speed_level、volume
-(NSString *)getTTSParam:(NSString *)param;

Meta 文件相关

/// 初始化meta文件及result文件计时器
-(void)startInitTime;

/// 保存Meta文件
/// @param fileName 文件名，需带后缀：xxx.meta
/// @param filePath 文件路径，如果为nil或@""，则存在默认路径
-(NSString*)saveMetaWithfileName:(NSString*)fileName andfilePath:(NSString*)filePath;

/// @param sequence 可理解为：章节
- (void)startSegment:(NSString *)sequence;

/// @param idCard 给某个章节添加身份证号
- (void)addSegment:(NSString *)sequence andIDCard:(NSString*)idCard;

- (void)endSegment:(NSString *)sequence;

/// @param actionName 开始某个章节的actionName检测项
- (void)startActionInSegment:(NSString *)sequence
        actionName:(NSString *)actionName;

- (void)endActionInSegment:(NSString *)sequence
        actionName:(NSString *)actionName;

/// @param word 识别激活词
- (void)startAddWord:(NSString *)word;

- (void)endAddWord:(NSString *)word;

/// @param label 人物名称
/// @param imageBase64 人脸照片的base64
- (void)addFace:(NSString *)label
        image:(NSString *)imageBase64;


/// @param holderIdCardNumber 身份证号
/// @param title 文档标题
- (void)addPolicy:(NSString *)holderIdCardNumber
        title:(NSString *)title;

/// @param detection 多端检测统一码
-(void)addDetection:(NSString*)detection;

result 文件相关

/// 开始某章某节的检测项
/// @param sequence 第几章
/// @param action 第几小节（用于区分同一章节多个相同的检测）
/// @param actionName 检测项名称
- (void)resultStartActionInSegment:(NSString *)sequence
                      action:(NSString *)action
                        actionName:(NSString *)actionName;

- (void)resultEndActionInSegment:(NSString *)sequence
                          action:(NSString *)action;

/// 给某一章某一节添加结果
/// @param sequence 第几章
/// @param action 第几小节
/// @param result 检测结果
- (void)resultAddActionInSegment:(NSString *)sequence
                action:(NSString *)action
                          result:(NSString *)result;

/// 给某一章某一节添加照片（base64）
/// @param sequence 第几章
/// @param action 第几小节
/// @param image 图片（暂时只获取身份证图片）
- (void)resultAddActionInSegment:(NSString *)sequence
                    action:(NSString *)action
                           image:(NSString *)image;

/// @param md5String 录制视频的md5加密
- (void)resultAddVideoMD5:(NSString *)md5String;

/// @param detection 统一标识码
- (void)resultAddDetection:(NSString *)detection;

/// 获取resultJson
- (NSString *)resultGetMetaJson;

/// 保存result文件
/// @param fileName 文件名，需带后缀：xxx.result.meta
/// @param filePath 文件路径，如果为nil或@""，则存在默认路径
- (NSString *)resultSaveToFile:(NSString *)fileName andPath:(NSString*)filePath;

tools

/// 获取SDK版本号
+ (NSString*)getVersion;

/// 将相机中获取的sampleBuffer保存到相册
/// @param sampleBuffer CMSampleBufferRef
- (void)saveSampleBufferFromCamera: (CMSampleBufferRef)sampleBuffer;

/// 将rtc视频流转成图片
/// @param imageBuffer CVPixelBufferRef、CVPixelBufferRef
- (UIImage *) getImageFromRPVideo: (CVImageBufferRef)imageBuffer;

释放

/// 释放资源
- (void)releaseResources;

初始化 RTC 实例

[MPIDRSSDK initRTCWithUserId:self.uid appId:AppId success:^(id  _Nonnull responseObject) {
    
    self.artvcEgnine = responseObject;
    
    self.artvcEgnine.delegate = self;
    self.artvcEgnine.videoProfileType = ARTVCVideoProfileType_640x360_15Fps;
    // 发布
    ARTVCPublishConfig *publishConfig = [[ARTVCPublishConfig alloc] init];
    publishConfig.videoProfile = self.artvcEgnine.videoProfileType;
    publishConfig.audioEnable = YES;
    publishConfig.videoEnable = YES;
    self.artvcEgnine.autoPublishConfig = publishConfig;
    self.artvcEgnine.autoPublish = YES;
    // 订阅
    ARTVCSubscribeOptions *subscribeOptions = [[ARTVCSubscribeOptions alloc] init];
    subscribeOptions.receiveAudio = YES;
    subscribeOptions.receiveVideo = YES;
    self.artvcEgnine.autoSubscribe = subscribeOptions;
    self.artvcEgnine.autoSubscribe = YES;
    
    //订阅本地视频流和音频流
    self.artvcEgnine.enableAudioBufferOutput = YES;
    self.artvcEgnine.enableCameraRawSampleOutput = YES;
    
    self.artvcEgnine.expectedAudioPlayMode = ARTVCAudioPlayModeSpeaker;
    self.artvcEgnine.degradationPreference = ARTVCDegradationPreferenceMAINTAIN_FRAMERATE;
    [self.artvcEgnine startCameraPreviewUsingBackCamera:NO];
    
    if (self.roomId && self.rtoken) {
        self.isOther = false;
        [IDRSSDK joinRoom:self.roomId token:self.rtoken];
    }else {
        self.isOther = true;
        [IDRSSDK createRoom];
    }
    
} failure:^(NSError * _Nonnull error) {
    
}];

视频相关检测

// 此方法为ARTVC 实例代理方法，详见：ARTVCEngineDelegate
- (void)didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer {
    __weak __typeof(self) weakSelf = self;
    dispatch_queue_t testqueue = dispatch_queue_create("subVideo", NULL);
    dispatch_sync(testqueue, ^{
        
        CVPixelBufferRef newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) ;
        
        if (_isWrite) {//签名检测；
            UIImage * image = [self.idrs getImageFromRPVideo:newBuffer];
            NSArray<NSNumber*> *kXMediaOptionsROIKey = @[@(0.2),@(0.2),@(0.6),@(0.6)];
            NSArray<IDRSSignConfidenceCheck *>*sings = [self.idrs checkSignClassifyWithImage:image AndROI:kXMediaOptionsROIKey];
            dispatch_async(dispatch_get_main_queue(), ^{
                NSString*string = @"";
                float max = 0.0;
                for (IDRSSignConfidenceCheck* Confidence in sings) {
                    if (Confidence.confidence > max) {
                        string = Confidence.label;
                        max = Confidence.confidence;
                    }
                }
                if ([string isEqual:@"hand"] && max > 0.95) {
                    self.resultLabel.text = [NSString stringWithFormat:@"手写体"];
                    [self didsendMessage:@"检测到手写体" isOn:true type:0];
                }
            });
        }
        
      
        if(_isFaceCheck){
            @autoreleasepool {
                //用图片检测
                IDRSFaceDetectParam *detectParam = [[IDRSFaceDetectParam alloc]init];
                detectParam.dataType = IDRSFaceDetectInputTypePixelBuffer;
                detectParam.buffer = newBuffer;
                detectParam.inputAngle = 0;
                detectParam.outputAngle = 0;
                detectParam.faceNetType = 0;
                detectParam.supportFaceRecognition = false;
                detectParam.supportFaceLiveness = false;
                
                [_idrs faceTrackFromVideo:detectParam faceDetectionCallback:^(NSError *error, NSArray<FaceDetectionOutput *> *faces) {
                    //脸的标识框：
                    dispatch_async(dispatch_get_main_queue(), ^{
                        self.drawView.faceDetectView.detectResult = faces;
                    });
                }];
            }
        }else{
            dispatch_async(dispatch_get_main_queue(), ^{
                self.drawView.faceDetectView.detectResult = nil;
            });
        }//人脸框结束;
        
        if (weakSelf.isOCR_Front || weakSelf.isOCR_Back) {
            @autoreleasepool {
                BOOL isForont = weakSelf.isOCR_Front? true:false;
                // 2. ocr身份证。
                IDRSIDCardDetectParam *idCardParam = [[IDRSIDCardDetectParam alloc]init];
                idCardParam.dataType = IDRSIDCardInputTypePixelBuffer;
                idCardParam.buffer = newBuffer;
                NSArray<NSNumber*> *kXMediaOptionsROIKey = @[@(0.2),@(0.2),@(0.6),@(0.6)];
                IDCardDetectionOutput *ocrResult = [_idrs detectIDCard:idCardParam roiKey:kXMediaOptionsROIKey rotate:@(90) isFrontCamera:NO isDetectFrontIDCard:isForont];
                
                if (ocrResult!=nil) {
                    NSString *result = @"";
                    if (ocrResult.num.length > 0) {
                        result = ocrResult.num;
                    }else if (ocrResult.date.length > 0){
                        result = ocrResult.date;
                    }
                    // 采集到了身份证信息
                    dispatch_async(dispatch_get_main_queue(), ^{
                        // 显示身份证信息
                        self.resultLabel.text = result;
                        [self didsendMessage:result isOn:true type:0];
                    });
                }
            }
        }
        
        // 3. 手势检测
        if (weakSelf.isHandDet) {
            @autoreleasepool {
                IDRSHandDetectParam *handParam = [[IDRSHandDetectParam alloc]init];
                handParam.dataType = IDRSHandInputTypeRGBA;
                handParam.buffer = newBuffer;
                handParam.outAngle = 0;
                NSArray<HandDetectionOutput *> *handResults = [_idrs detectHandGesture:handParam];
                
                if(handResults.count > 0) {
                    // 判断签字
                    if (handResults[0].phone_touched_score>0) {
                        //手写框
                        dispatch_async(dispatch_get_main_queue(), ^{
                            weakSelf.drawView.handDetectView.detectResult = handResults;
                            if(handResults[0].hand_phone_action == 1 || handResults[0].hand_phone_action == 2) {
                                self.resultLabel.text = @"签字成功";
                                [self didsendMessage:@"签字成功" isOn:true type:0];
                            }
                        });
                    }
                }else{
                    dispatch_async(dispatch_get_main_queue(), ^{
                        weakSelf.drawView.handDetectView.detectResult = nil;
                    });
                }
            }
        }
        //静态手势检测
        if(self.isStaticHandDet) {
            @autoreleasepool {
                IDRSHandDetectParam *handParam = [[IDRSHandDetectParam alloc]init];
                handParam.dataType = IDRSHandInputTypeRGBA;
                handParam.buffer = newBuffer;
                handParam.outAngle = 0;
                
                NSArray<HandDetectionOutput *> *handResults = [_idrs detectHandStaticGesture:handParam];
                dispatch_async(dispatch_get_main_queue(), ^{
                    if (handResults.count > 0) {
                        weakSelf.drawView.handDetectView.detectResult = handResults;
                        
                        HandDetectionOutput *handResult = handResults[0];
                        if (handResult.hand_action_type == 0 && handResult.hand_static_action > 0) {
                            NSString *result = [self handStaticActionTypeToText:handResult.hand_static_action];
                            self.resultLabel.text = result;
                            [self didsendMessage:result isOn:true type:0];
                            
                        }
                    }else{
                        dispatch_async(dispatch_get_main_queue(), ^{
                            weakSelf.drawView.handDetectView.detectResult = nil;
                        });
                    }
                });
            }
        }
        if (!weakSelf.isHandDet && !weakSelf.isStaticHandDet) {
            dispatch_async(dispatch_get_main_queue(), ^{
                weakSelf.drawView.handDetectView.detectResult = nil;
            });
        }
    });
}

音频相关检测

// 此方法为ARTVC 实例代理方法，详见：ARTVCEngineDelegate
- (void)didOutputAudioBuffer:(ARTVCAudioData*)audioData {
    if (_isNui) {
        if (audioData.audioBufferList->mBuffers[0].mData != NULL && audioData.audioBufferList->mBuffers[0].mDataByteSize > 0) {
            //            NSData *srcData = [NSData dataWithBytes:audioData.audioBufferList->mBuffers[0].mData length:audioData.audioBufferList->mBuffers[0].mDataByteSize];
            //重采样为16K
            pcm_frame_t pcmModelInput;
            pcmModelInput.len = audioData.audioBufferList->mBuffers[0].mDataByteSize;
            pcmModelInput.buf =  (uint8_t*)audioData.audioBufferList->mBuffers[0].mData;
            pcmModelInput.sample_rate = audioData.sampleRate;
            
            pcm_frame_t pcmModelOutput;
            pcm_resample_16k(&pcmModelInput, &pcmModelOutput);
            
            NSData *srcData = [NSData dataWithBytes:pcmModelOutput.buf length:pcmModelOutput.len];
            [self.idrs feedAudioFrame:srcData];
        }
    }
}

录制功能

说明

成功初始化 MPIDRSSDK 和 ARTVC 实例以后调用才生效。

设置水印并开启远程录制

[MPIDRSSDK openRemoteRecordWithRoomId:self.roomId
                              waterMarkId:self.watermarkId];

关闭远程录制

+ (void)stopRemoteRecord:(NSString *)recordId;

录制回调

- (void)didReceiveCustomSignalingResponse:(NSDictionary *)dictionary{
    id opcmdObject = [dictionary objectForKey:@"opcmd"];
    if ([opcmdObject isKindOfClass:[NSNumber class]]) {
        int opcmd = [opcmdObject intValue];
        switch (opcmd) {
            case 1032: {
                NSLog(@"开始录制");
                self.startTime = [NSDate date];
                self.recordId = [dictionary objectForKey:@"recordId"];
            }
                break;
            case 1034: {
                NSLog(@"结束录制");
                self.duration = [[NSDate date] timeIntervalSince1970] - [self.startTime timeIntervalSince1970];
            }
                break;
            default:
                break;
        } }
}

上传录制产物

self.duration = [[NSDate date] timeIntervalSince1970] - [self.startTime timeIntervalSince1970];
IDRSUploadManagerParam *param = [[IDRSUploadManagerParam alloc] init];
param.duration = self.duration;
param.appId = AppId;
param.ak = Ak;
param.sk = Sk;
param.type = IDRSRecordRemote;
param.recordAt = self.startTime;
param.roomId = self.roomId;

[IDRSUploadManager uploadFileWithParam:param success:^(id  _Nonnull responseObject) {
    [self showToastWith:responseObject duration:3];
} failure:^(NSError * _Nonnull error, IDRSUploadManagerParam * _Nonnull upLoadParam) {
    //
    if (upLoadParam) {
        [self showToastWith:@"upload error" duration:3];
        
    }
}];

首页 智能双录质检 SDK参考 iOS SDK 远程双录