Harmony集成概览

步骤一：申请APP音视频权限

在entry模块的module.json配置APP长时间后台模式，添加麦克风、摄像头、长时间后台任务授权。

...  
"abilities": [
  {
    "name": "xxx",
    "backgroundModes": [
      "audioPlayback",
      "audioRecording",
      "dataTransfer"
    ],
    ...
  }
],
"requestPermissions": [
  {
    "name": "ohos.permission.MICROPHONE",
    "reason": "$string:module_desc",
    "usedScene": {
      "abilities": [
        "EntryAbility"
      ],
      "when": "always"
    }
  },
  {
    "name": "ohos.permission.CAMERA",
    "reason": "$string:reason_camera",
    "usedScene": {
      "abilities": [
        "EntryAbility"
      ],
      "when": "always"
    }
  },
  {
    "name": "ohos.permission.KEEP_BACKGROUND_RUNNING",
    "reason": "$string:keep_background",
    "usedScene": {
      "abilities": [],
      "when": "always"
    }
  }
]

在发起通话前，检查授权情况，如未授权，则弹框交由用户进行授权，需要您在App上自行实现。

const permissions: Array<Permissions> = ['ohos.permission.MICROPHONE','ohos.permission.CAMERA','ohos.permission.KEEP_BACKGROUND_RUNNING'];
function reqPermissionsFromUser(permissions: Array<Permissions>, context: common.UIAbilityContext): void {
  let atManager: abilityAccessCtrl.AtManager = abilityAccessCtrl.createAtManager();
  // requestPermissionsFromUser会判断权限的授权状态来决定是否唤起弹窗
  atManager.requestPermissionsFromUser(context, permissions).then((data) => {
    let grantStatus: Array<number> = data.authResults;
    let length: number = grantStatus.length;
    for (let i = 0; i < length; i++) {
      if (grantStatus[i] === 0) {
        // 用户授权，可以继续访问目标操作
      } else {
        // 用户拒绝授权，提示用户必须授权才能访问当前页面的功能，并引导用户到系统设置中打开相应的权限
        return;
      }
    }
    // 授权成功
  }).catch((err: BusinessError) => {
    console.error(`Failed to request permissions from user. Code is ${err.code}, message is ${err.message}`);
  })
}

步骤二：导入ARTCAICall相关类

从@aliyun_video_cloud/alivcsdk_artc里按需导入 ARTCAICall 相关的类和接口：

import {
  ARTCAICallEngineListener,
  ARTCAICallEngineInterface,
  ARTCAICallEngineFactory,
  ARTCAICallAgentType,
  ARTCAICallConfig,
  ARTCAICallUtils,
  ARTCAICallRTCTokenHelper,
  ARTCAICallLog,
  ARTCAICallLogType,
  ARTCAICallErrorCode,
  ARTCAICallConnectionStatus,
  ARTCAICallAgentState,
  ARTCAICallNetworkQuality,
  ARTCAICallAgentSubtitleResult,
  ARTCAICallUserSubtitleResult,
  ARTCAICallSpeakingInterruptedReason,
  RecordValueType,
  ARTCAICallAgentVcrResult,
  ARTCAICallAudioAccompanyState,
  ARTCAICallAudioAccompanyErrorCode,
  ARTCAICallViewConfig,
  ARTCAICallVideoConfig,
  ARTCAICallAgentInfo,
} from '@aliyun_video_cloud/alivcsdk_artc';

步骤三：创建&初始化引擎

创建&初始化ARTCAICallEngine引擎，示例代码如下：

// 定义引擎接口
private aiCallSDK: ARTCAICallEngineInterface | undefined = undefined;

// 创建引擎
this.aiCallSDK = ARTCAICallEngineFactory.createEngine(getContext(this));

步骤四：实现回调方法

按需实现引擎回调方法。引擎回调接口详情参考：API接口详情。

const listener = new ARTCAICallEngineListener();
listener.onErrorOccursCallback = (code: ARTCAICallErrorCode) => {
  // 发送了错误
  this.handup()
};
listener.onCallBeginCallback = () => {
  // 开始通话，这里可以打印智能体实例信息，可以用于排障
  const agentInfo = this.aiCallSDK?.agentInfo!;
  ARTCAICallLog.writeLog(ARTCAICallLogType.Info, `onCallBeginCallback: AgentInfo {
        instanceId: ${ agentInfo.instanceId },
        agentId: ${ agentInfo.agentId },
        agentType: ${ agentInfo.agentType },
        channelId: ${ agentInfo.channelId },
        uid: ${ agentInfo.uid },
        requestId: ${ agentInfo.requestId },
        region:, ${ agentInfo.region }
      }`);
};
listener.onCallEndCallback = () => {
  // 结束通话
};
listener.onAgentStateChangedCallback = (state: ARTCAICallAgentState) => {
  // 智能体状态改变
};
listener.onUserSubtitleNotifyCallback = (data: ARTCAICallUserSubtitleResult) => {
// 用户提问被智能体识别结果的通知
};
listener.onVoiceAgentSubtitleNotifyCallback = (data: ARTCAICallAgentSubtitleResult) => {
  // 智能体回答结果通知
};

listener.onVoiceInterruptedCallback = (enable: boolean) => {
  // 当前通话的语音打断是否启用
};
...

this.aiCallSDK.listener = listener;

步骤五：创建并初始化ARTCAICallConfig

ARTCAICallConfig详情，请参见ARTCAICallConfig。

private agentType: ARTCAICallAgentType = ARTCAICallAgentType.VoiceAgent

const aiCallConfig = new ARTCAICallConfig();
aiCallConfig.agentId = 'xxx';              // 智能体Id
aiCallConfig.agentType = this.agentType;   // 智能体类型
aiCallConfig.region = 'xx-xxx';            // 智能体服务所在的区域
aiCallConfig.userId = 'xxx';               // 推荐使用你的App登录后的用户id
aiCallConfig.userJoinToken = 'xxxxxxxxx';  // RTC Token

// 针对视觉理解&视频通话，需要设置视频配置
// 这里frameRate设置为5，需要根据控制台上的智能体的抽帧率（一般为2）进行调整，最大不建议超过15fps
// bitrate: frameRate超过10可以设置为512
if (this.agentType === ARTCAICallAgentType.VisionAgent || this.agentType === ARTCAICallAgentType.VideoAgent) {
  const videoConfig = new ARTCAICallVideoConfig();
  videoConfig.useFrontCameraDefault = true;
  videoConfig.frameRate = 5;
  videoConfig.bitrate = 340;
  this.aiCallSDK.videoConfig = videoConfig;
}

步骤六：【可选】创建并初始化渲染视图

如果不是语音通话智能体，那么需要对本地摄像头采集预览或对远端数字人进行渲染。

private localViewConfig = new ARTCAICallViewConfig();
private agentViewConfig = new ARTCAICallViewConfig();

// 对于数字人&视频通话智能体类型，需要在Builder里增加XComponent用于智能体预览
XComponent({
  id: 'Agent_XComponent',
  type: XComponentType.SURFACE,
  controller: this.agentViewConfig.controller
})

// 对于视觉理解&视频通话智能体类型，在Builder里增加XComponent用于本地采集预览
XComponent({
  id: 'Local_XComponent',
  type: XComponentType.SURFACE,
  controller: this.localViewConfig.controller
})


// 对于数字人&视频通话智能体类型，设置数字人视图渲染，需要在XComponent加载完成后调用
if (this.agentType === ARTCAICallAgentType.AvatarAgent || this.agentType === ARTCAICallAgentType.VideoAgent) {
  this.aiCallSDK.setAgentViewConfig(this.agentViewConfig);
}
// 对于视觉理解&视频通话智能体类型，设置本地视图渲染，需要在XComponent加载完成后调用
if (this.agentType === ARTCAICallAgentType.VisionAgent || this.agentType === ARTCAICallAgentType.VideoAgent) {
  this.aiCallSDK.setLocalViewConfig(this.localViewConfig);
}

步骤七：发起智能体呼叫

调用callWithConfig接口发起智能体呼叫。

const result = this.aiCallSDK.callWithConfig(aiCallConfig);
if (result === true) {
  // 成功
}

步骤八：通话中的业务实现

在开启通话后，您可以根据您的业务需求处理字幕、打断智能体讲话等。

实时字幕

智能体状态

打断智能体讲话

对讲机模式

截帧质检

步骤九：通话结束，挂断智能体通话

调用handup()接口，挂断智能体通话。

handup = () => {
  if (this.aiCallSDK === undefined) {
    return;
  }
  this.aiCallSDK.handup(true);
  this.aiCallSDK = undefined;
}