版本变更
版本 | 描述 | 时间 |
v0.1 | 功能发布 | 2024-07-01 |
概览
交互示例
接入准备
需要接入方提前准备阿里云账号,并利用阿里云子账号生成对应的AK/SK;
阿里云主账号需要对生成AK/SK的子账号进行RAM授权;
使用阿里云主账号登录平台,签署相关法务协议。
API详情
1. 查询数字人项目信息
入参 QueryAvatarProjectRequest
参数名 | 类型 | 是否必填 | 说明 |
projectId | String | Y | 项目ID |
出参 QueryAvatarProjectResponse
参数名 | 类型 | 说明 |
status | String | 启动结果:DEPLOYING - 发布中 DEPLOYED - 已发布 DEPLOY_FAIL - 发布失败 |
projectName | String | 名称 |
agentId | String | 智能体id |
errorMsg | String | 发布失败原因 |
2. 启动会话
入参 StartAvatarSessionRequest
参数名 | 类型 | 是否必填 | 说明 |
projectId | String | Y | 项目ID |
requestId | String | Y | 请求id用于幂等 |
出参 StartAvatarSessionResponse
参数名 | 类型 | 说明 |
sessionId | String | 会话id |
channelToken | String | 频道信息(json格式) |
channelToken 字段解析
{
"channelId":"123",//频道ID
"token":"", // 令牌
"expireTime":600,//过期时间(单位秒)
"nonce":"",//随机数
"userId":"",//用户ID
"appId":""//应用ID
}
3. 停止会话
入参 StopAvatarSessionRequest
参数名 | 类型 | 是否必填 | 说明 |
projectId | String | Y | 项目ID |
sessionId | String | Y | 会话ID |
出参 StopAvatarSessionResponse
参数名 | 类型 | 说明 |
status | String | 停止结果:Stopped - 已停止 StoppedFail - 停止失败 |
4. 检查会话状态
入参 CheckSessionRequest
参数名 | 类型 | 是否必填 | 说明 |
projectId | String | Y | 项目ID |
sessionId | String | Y | 会话ID |
出参 CheckSessionResponse
参数名 | 类型 | 说明 |
status | String | 会话状态:FREE - 空闲 BUSY - 忙线 |
5. 发送文本消息
入参 SendTextMsgRequest
参数名 | 类型 | 是否必填 | 说明 |
projectId | String | Y | 项目ID |
requestId | String | Y | 唯一请求ID |
sessionId | String | Y | 会话ID |
text | String | Y | 文本信息 |
type | Integer | Y | 消息类型: 1 - 提问 2- 回答 |
出参 SendTextMsgResponse
参数名 | 类型 | 说明 |
status | String | 状态:SUCCESS - 发送成功, FAIL-发送失败 |
6. 有效资源查询
入参 QueryAvatarResourceRequest
参数名 | 类型 | 是否必填 | 说明 |
出参 QueryAvatarResourceResponse
参数名 | 类型 | 说明 |
queryResourceInfoList | List<QueryResourceInfo> | 资源信息 |
QueryResourceInfo
参数名 | 类型 | 说明 |
resourceId | String | 资源id |
type | String | 资源类型: STANDARD - 2D数字人实时互动【基础版】 ADVANCED - 2D数字人实时互动【高级版】 |
validPeriodTime | String | 有效期(时间戳) 例如:1719904342237 |
对接详情
PHP
对接示例
require 'vendor/autoload.php';
use AlibabaCloud\SDK\Imarketing\V20220704\Models\GetOssUploadSignatureRequest;
use AlibabaCloud\SDK\IntelligentCreation\V20240313\IntelligentCreation;
use Darabonba\OpenApi\Models\Config as AlibabaConfig;
$config = new AlibabaConfig();
$config->accessKeyId = '****';
$config->accessKeySecret = '****';
$config->endpoint = "intelligentcreation.cn-zhangjiakou.aliyuncs.com";
$intelligentCreationClient = new IntelligentCreation($config);
$request = new QueryAvatarProjectRequest();
$request->projectId = '111';
try {
$response = $intelligentCreationClient->queryAvatarProject($request);
var_dump($response->toMap());
} catch (TeaError $e) {
Log::error($e);
}
2.1.4
u-ce29b5c7-8a37-4590-98f4-71a02e5366f9-composer-tea.zip
composer require alibabacloud/intelligentcreation-20240313 2.1.4
Java
对接示例
package com.aliyun.intelligentcreation20240313;
import com.aliyun.intelligentcreation20240313.models.*;
import com.aliyun.tea.TeaException;
import com.aliyun.teaopenapi.models.Config;
import com.google.gson.Gson;
import java.util.HashMap;
import java.util.Map;
public class TestAvatarTest {
public TestAvatarTest() throws Exception {
}
public static void main(String[] args) throws Exception {
TestAvatarTest avatarTest = new TestAvatarTest();
try {
String projectId = "780931376329506816";
avatarTest.queryAvatarProjectTest(projectId);
String sessionId = avatarTest.startAvatarSessionRequest(projectId);
avatarTest.checkSessionRequest(projectId,sessionId);
avatarTest.sendTextMsgRequest(projectId,sessionId);
avatarTest.stopAvatarSessionRequest(projectId,sessionId);
avatarTest.queryAvatarResourceRequest(projectId);
} catch (TeaException e) {
Gson gson = new Gson();
System.out.println(e.getMessage());
System.out.println(gson.toJson(e.getData()));
}
}
String url = "intelligentcreation.cn-zhangjiakou.aliyuncs.com";
//初始化配置
String ak = "**";
String sk = "**";
Config config = new Config().setAccessKeyId(ak)
.setAccessKeySecret(sk)
.setEndpoint(url);
// 创建客户端
Client client = new Client(config);
void queryAvatarProjectTest(String projectId) throws Exception {
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>();
map.put("projectId",projectId);
QueryAvatarProjectRequest request = QueryAvatarProjectRequest.build(map);
// 请求接口
QueryAvatarProjectResponse response = client.queryAvatarProject(request);
System.out.println(gson.toJson(response));
if (response.getStatusCode().equals(200)) {
System.out.println("queryAvatarProjectTest 请求成功");
}
}
String startAvatarSessionRequest(String projectId) throws Exception {
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>();
map.put("projectId",projectId);
StartAvatarSessionRequest startAvatarSessionRequest = StartAvatarSessionRequest.build(map);
StartAvatarSessionResponse response = client.startAvatarSession(startAvatarSessionRequest);
System.out.println(gson.toJson(response));
if (response.getStatusCode().equals(200)) {
System.out.println("startAvatarSessionRequest 请求成功");
return response.getBody().getSessionId();
}
return null;
}
void stopAvatarSessionRequest(String projectId, String sessionId) throws Exception {
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>();
map.put("projectId",projectId);
map.put("sessionId",sessionId);
StopAvatarSessionRequest request = StopAvatarSessionRequest.build(map);
StopAvatarSessionResponse response = client.stopAvatarSession(request);
System.out.println(gson.toJson(response));
if (response.getStatusCode().equals(200)) {
System.out.println("stopAvatarSessionRequest 请求成功");
}
}
void checkSessionRequest(String projectId, String sessionId) throws Exception {
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>();
map.put("projectId",projectId);
map.put("sessionId",sessionId);
CheckSessionRequest request = CheckSessionRequest.build(map);
CheckSessionResponse response = client.checkSession(request);
System.out.println(gson.toJson(response));
if (response.getStatusCode().equals(200)) {
System.out.println("checkSessionRequest 请求成功");
}
}
void sendTextMsgRequest(String projectId,String sessionId) throws Exception {
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>();
map.put("projectId",projectId);
map.put("sessionId",sessionId);
map.put("requestId",System.currentTimeMillis());
map.put("text","test sendTextMsgRequest");
map.put("type",1);
SendTextMsgRequest request = SendTextMsgRequest.build(map);
SendTextMsgResponse response = client.sendTextMsg(request);
System.out.println(gson.toJson(response));
if (response.getStatusCode().equals(200)) {
System.out.println("sendTextMsgRequest 请求成功");
}
}
void queryAvatarResourceRequest(String projectId) throws Exception {
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>();
QueryAvatarResourceRequest request = QueryAvatarResourceRequest.build(map);
QueryAvatarResourceResponse response = client.queryAvatarResource(request);
System.out.println(gson.toJson(response));
if (response.getStatusCode().equals(200)) {
System.out.println("queryAvatarResourceRequest 请求成功");
}
}
}
2.1.0
u-ce29b5c7-8a37-4590-98f4-71a02e5366f9-java-tea.zip
<dependency>
<groupId>com.aliyun</groupId>
<artifactId>intelligentcreation20240313</artifactId>
<version>2.1.0</version>
</dependency>
安卓
实时音视频SDK接入文档: 实时音频快速入门指南-阿里云帮助中心
※ mLocalSurfaceContainer 是 视频流的展示容器
※ projectId 是项目Id
添加maven仓库
maven(url = "https://maven.aliyun.com/nexus/content/repositories/releases")
添加推流SDK依赖
implementation("com.aliyun.aio:AliVCSDK_ARTC:6.8.7")
初始化拉流引擎
private fun createEngine() {
mAliRtcEngine = AliRtcEngine.getInstance(applicationContext)
mAliRtcEngine?.setRtcEngineEventListener(object : AliRtcEngineEventListener() {
/* SDK与服务器的链接状态通知,务必处理链接失败的情况 */
override fun onConnectionStatusChange(
aliRtcConnectionStatus: AliRtcEngine.AliRtcConnectionStatus,
aliRtcConnectionStatusChangeReason: AliRtcEngine.AliRtcConnectionStatusChangeReason
) {
super.onConnectionStatusChange(
aliRtcConnectionStatus,
aliRtcConnectionStatusChangeReason
)
if (aliRtcConnectionStatus == AliRtcEngine.AliRtcConnectionStatus.AliRtcConnectionStatusFailed) {
/* TODO: 务必处理;建议业务提示客户,此时SDK内部已经尝试了各种恢复策略已经无法继续使用时才会上报 */
} else {
/* TODO: 可选处理;增加业务代码,一般用于数据统计、UI变化 */
}
}
/* SDK尝试控制本地设备异常 */
override fun OnLocalDeviceException(
aliRtcEngineLocalDeviceType: AliRtcEngine.AliRtcEngineLocalDeviceType,
aliRtcEngineLocalDeviceExceptionType: AliRtcEngine.AliRtcEngineLocalDeviceExceptionType,
s: String
) {
//TODO 发生该异常时App需要检测权限、设备硬件是否正常。
}
override fun onJoinChannelResult(
result: Int,
channel: String,
userId: String,
elapsed: Int
) {
super.onJoinChannelResult(result, channel, userId, elapsed)
Log.i(TAG, "onJoinChannelResult result=$result,channel=$channel,userId=$userId,elapsed=$elapsed")
}
override fun onLeaveChannelResult(result: Int, stats: AliRtcEngine.AliRtcStats) {
super.onLeaveChannelResult(result, stats)
Log.i(TAG, "onLeaveChannelResult result=$result")
}
})
mAliRtcEngine?.setRtcEngineNotify(object : AliRtcEngineNotify() {
/* 鉴权距离过期还有30s时会回调,务必进行鉴权时间刷新 */
override fun onAuthInfoWillExpire() {
super.onAuthInfoWillExpire()
/* TODO: 务必处理;业务触发重新获取当前channel,user的鉴权信息,然后设置refreshAuthInfo即可 */
}
/* 业务可能会触发踢人的动作,所以这个地方也需要处理 */
override fun onBye(code: Int) {
super.onBye(code)
/* TODO: 建议业务根据自己的场景,进行对应的处理 */
}
override fun onRemoteUserOnLineNotify(uid: String, elapsed: Int) {
super.onRemoteUserOnLineNotify(uid, elapsed)
Log.i(TAG, "onRemoteUserOnLineNotify uid=$uid,elapsed=$elapsed")
}
override fun onRemoteUserOffLineNotify(
uid: String,
aliRtcUserOfflineReason: AliRtcEngine.AliRtcUserOfflineReason
) {
super.onRemoteUserOffLineNotify(uid, aliRtcUserOfflineReason)
Log.i(TAG, "onRemoteUserOnLineNotify uid=$uid,aliRtcUserOfflineReason=$aliRtcUserOfflineReason")
}
override fun onRemoteTrackAvailableNotify(
uid: String,
aliRtcAudioTrack: AliRtcEngine.AliRtcAudioTrack,
aliRtcVideoTrack: AliRtcEngine.AliRtcVideoTrack
) {
super.onRemoteTrackAvailableNotify(uid, aliRtcAudioTrack, aliRtcVideoTrack)
Log.i(TAG, "onRemoteUserOnLineNotify uid=$uid,aliRtcAudioTrack=$aliRtcAudioTrack,aliRtcVideoTrack=$aliRtcVideoTrack")
mLocalSurfaceContainer.post{
if (aliRtcVideoTrack == AliRtcEngine.AliRtcVideoTrack.AliRtcVideoTrackCamera
|| aliRtcVideoTrack == AliRtcEngine.AliRtcVideoTrack.AliRtcVideoTrackBoth
) {
val remote_canvas = AliRtcVideoCanvas()
val remoteView = mAliRtcEngine?.createRenderSurfaceView(this@PreviewActivity)
if (remoteView != null) {
remoteView.setZOrderOnTop(true)
remoteView.setZOrderMediaOverlay(true)
}
remote_canvas.view = remoteView
mLocalSurfaceContainer.addView(
remote_canvas.view, FrameLayout.LayoutParams(
mLocalSurfaceContainer.width,
mLocalSurfaceContainer.height
)
)
mAliRtcEngine?.setRemoteViewConfig(
remote_canvas,
uid,
AliRtcEngine.AliRtcVideoTrack.AliRtcVideoTrackCamera
)
} else {
mAliRtcEngine?.setRemoteViewConfig(
null,
uid,
AliRtcEngine.AliRtcVideoTrack.AliRtcVideoTrackCamera
)
}
}
}
})
}
入会前引擎参数配置
private fun initEngineBeforeJoin() {
/* 可选:入会前的参数设置 */
mAliRtcEngine?.setChannelProfile(AliRtcEngine.AliRTCSdkChannelProfile.AliRTCSdkInteractiveLive)
mAliRtcEngine?.setClientRole(AliRtcEngine.AliRTCSdkClientRole.AliRTCSdkLive)
/* 设置音频的属性 */
mAliRtcEngine?.setAudioProfile(
AliRtcEngine.AliRtcAudioProfile.AliRtcEngineStereoHighQualityMode,
AliRtcEngine.AliRtcAudioScenario.AliRtcSceneMusicMode
)
/* 可选:摄像头预览,不设置也会进行推流 */
val canvas = AliRtcVideoCanvas()
canvas.view = mAliRtcEngine?.createRenderSurfaceView(this)
mLocalSurfaceContainer.removeAllViews()
if (canvas.view != null) {
mLocalSurfaceContainer.addView(
canvas.view,
FrameLayout.LayoutParams(
FrameLayout.LayoutParams.MATCH_PARENT,
FrameLayout.LayoutParams.MATCH_PARENT
)
)
}
mAliRtcEngine?.setLocalViewConfig(
canvas,
AliRtcEngine.AliRtcVideoTrack.AliRtcVideoTrackCamera
)
}
初始化pop接口SDK
private fun initPopApiClient(){
val config = Config()
.setAccessKeyId(KEY_ID)
.setAccessKeySecret(KEY_SECRET)
.setEndpoint(HOST)
client = Client(config)
}
获取项目信息
private suspend fun getProjectInfo() = withContext(Dispatchers.IO) {
val map: Map<String , String> = mutableMapOf<String, String>().apply {
put(PROJECT_ID , projectId?:"")
}
client?.queryAvatarProject(QueryAvatarProjectRequest.build(map))?.apply {
Log.e(TAG , "statusCode:${statusCode}")
if (statusCode == 200) {
Log.e(TAG , "projectName:${body.projectName}")
tvProjectName.post {
tvProjectName.text = body.projectName
}
}
}
}
获取会议信息
private suspend fun getMeetingInfo() = withContext(Dispatchers.IO){
val map: Map<String , String> = mutableMapOf<String, String>().apply {
put(PROJECT_ID , projectId?:"")
}
val response: StartAvatarSessionResponse? = client?.startAvatarSession(StartAvatarSessionRequest.build(map))
response?.apply {
Log.e(TAG , "statusCode:${statusCode}")
if (statusCode == 200) {
Log.e(TAG , "sessionId:${body.sessionId},channelToken:${body.channelToken}")
sessionId = body.sessionId
val meetingJsonObj = JSONObject(body.channelToken)
var channelId = ""
var token = ""
var nonce = ""
var userId = ""
var appId = ""
var expireTime = 0L
var gslbList:MutableList<String> = mutableListOf()
if (meetingJsonObj.has(CHANNEL_ID)){
channelId = meetingJsonObj.optString(CHANNEL_ID)
}
if (meetingJsonObj.has(TOKEN)){
token = meetingJsonObj.optString(TOKEN)
}
if (meetingJsonObj.has(NONCE)){
nonce = meetingJsonObj.optString(NONCE)
}
if (meetingJsonObj.has(USER_ID)){
userId = meetingJsonObj.optString(USER_ID)
}
if (meetingJsonObj.has(APP_ID)){
appId = meetingJsonObj.optString(APP_ID)
}
if (meetingJsonObj.has(EXPIRE_TIME)){
expireTime = meetingJsonObj.optLong(EXPIRE_TIME)
}
if (meetingJsonObj.has(GSLB_LIST)) {
val gslbListJson = meetingJsonObj.optJSONArray(GSLB_LIST)
for (i in 0 until gslbListJson.length()) {
gslbList.add(gslbListJson.optString(i))
}
}
join(appId , channelId , userId , nonce,expireTime , token , gslbList.toTypedArray())
}
}
}
入会
private fun join(appId:String,channelId:String , userId:String ,
nonce:String , expireTime:Long,token:String,gslbList:Array<String>){
val userInfo = AliRtcAuthInfo()
userInfo.setAppId(appId)
userInfo.setChannelId(channelId)
userInfo.setUserId(userId)
userInfo.setNonce(nonce)
userInfo.setTimestamp(expireTime)
userInfo.setGslb(gslbList)
userInfo.setToken(token)
mAliRtcEngine?.joinChannel(userInfo, "testUserName")
resetCountDown()
}
发送问题
private fun send(){
val question = etInputQuestion.text.toString().trim()
if (TextUtils.isEmpty(question)) {
Toast.makeText(this , "question content cannot be null" , Toast.LENGTH_SHORT).show()
return
}
GlobalScope.launch {
sendText(question)
}
}
private suspend fun sendText(text:String) = withContext(Dispatchers.IO){
val map: Map<String , String> = mutableMapOf<String, String>().apply {
put(PROJECT_ID , projectId?:"")
put(SESSION_ID , sessionId)
put(REQUEST_ID , (System.currentTimeMillis()/1000).toString())
put(TEXT , text)
put(TYPE , "1")
}
client?.sendTextMsg(SendTextMsgRequest.build(map))?.apply {
onSendResponse(statusCode == 200 && body.status == "SUCCESS" , text)
}
}
private fun onSendResponse(result:Boolean , text:String){
tvExpireHint.post {
if (result) {
etInputQuestion.setText("")
questionAdapter?.data?.add(0 , text)
questionAdapter?.notifyItemInserted(0)
rvQuestion.scrollToPosition(0)
resetCountDown()
Toast.makeText(this@PreviewActivity , "发送成功" , Toast.LENGTH_SHORT).show()
}else {
Toast.makeText(this@PreviewActivity , "发送失败" , Toast.LENGTH_SHORT).show()
}
}
}
离会
private fun leave(){
mAliRtcEngine?.leaveChannel()
findViewById<LinearLayout>(R.id.llEnd).visibility = View.VISIBLE
val map: Map<String , String> = mutableMapOf<String, String>().apply {
put(PROJECT_ID , projectId?:"")
put(SESSION_ID , sessionId)
}
GlobalScope.launch {
async {
client?.stopAvatarSession(StopAvatarSessionRequest.build(map))?.apply {
Log.e(TAG , "statusCode = ${statusCode}, status = ${body.status}")
}
}
}
}
资源回收
override fun onDestroy() {
super.onDestroy()
mAliRtcEngine?.destroy()
mAliRtcEngine = null
handler.removeCallbacksAndMessages(null)
}
10min无提问,自动离会
private var handler: Handler = Handler(Looper.getMainLooper())
private val mRunnable = Runnable {
dealCountDown()
}
private fun dealCountDown(){
if (messageCount == 0) {
//离会
leave()
return
}
//小于1min ,提示
val timeStr = when {
60 == messageCount-> "01:00"
(messageCount in 1..9) -> "00:0${messageCount}"
else -> "00:${messageCount}"
}
llExpireHint.visibility = View.VISIBLE
tvExpireHint.text = "检测到您无操作,预览将于${timeStr}自动关闭,以避免占用路数"
messageCount--
handler.postDelayed(mRunnable , 1000)
}
//在入会和发送问题之后调用
private fun resetCountDown(){
handler.removeCallbacksAndMessages(null)
messageCount = 60
handler.postDelayed(mRunnable , 1000 * 60 * 9)
}