快速集成智能纪要Agent

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.alibaba.tingwu</groupId>
    <artifactId>test</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>com.aliyun</groupId>
            <artifactId>aliyun-java-sdk-core</artifactId>
            <version>4.6.4</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.74</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.30</version>
            <scope>provided</scope>
        </dependency>

        <!-- Java-WebSocket -->
        <dependency>
            <groupId>org.java-websocket</groupId>
            <artifactId>Java-WebSocket</artifactId>
            <version>1.5.3</version>
        </dependency>

        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter</artifactId>
            <version>5.9.3</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13.1</version>
            <scope>compile</scope>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>dashscope-sdk-java</artifactId>
            <version>2.21.16</version>
        </dependency>
        <dependency>
            <groupId>io.reactivex.rxjava2</groupId>
            <artifactId>rxjava</artifactId>
            <version>2.2.21</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.26</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-engine</artifactId>
            <version>5.10.2</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-api</artifactId>
            <version>5.10.2</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.junit-pioneer</groupId>
            <artifactId>junit-pioneer</artifactId>
            <version>1.9.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>io.javalin</groupId>
            <artifactId>javalin</artifactId>
            <version>4.4.0</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.9</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>2.0.7</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>2.0.7</version>
        </dependency>
        <dependency>
            <groupId>org.jetbrains.kotlin</groupId>
            <artifactId>kotlin-stdlib-jdk8</artifactId>
            <version>1.8.21</version>
        </dependency>
        <dependency>
            <groupId>com.squareup.okio</groupId>
            <artifactId>okio</artifactId>
            <version>3.6.0</version>
        </dependency>
        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>mockwebserver</artifactId>
            <version>4.12.0</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>logging-interceptor</artifactId>
            <version>4.12.0</version>
        </dependency>
        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>okhttp-sse</artifactId>
            <version>4.12.0</version>
        </dependency>
        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>okhttp</artifactId>
            <version>4.12.0</version>
        </dependency>
        <dependency>
            <groupId>com.github.victools</groupId>
            <artifactId>jsonschema-generator</artifactId>
            <version>4.31.1</version>
        </dependency>
    </dependencies>

</project>

import com.alibaba.dashscope.multimodal.MultiModalDialog;
import com.alibaba.dashscope.multimodal.MultiModalDialogCallback;
import com.alibaba.dashscope.multimodal.MultiModalRequestParam;
import com.alibaba.dashscope.multimodal.State;
import com.alibaba.dashscope.utils.Constants;
import com.alibaba.dashscope.utils.JsonUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.annotation.JSONField;
import com.google.gson.JsonObject;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;

import javax.sound.sampled.*;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static java.lang.Thread.sleep;

@Slf4j
public class MultiModalMeetingOfflineSDKDemo {
    @Data
        public static class CommandDTO {
            @JSONField(name = "intent_info")
            private IntentInfo intentInfo;
            @JSONField(name = "command_request_id")
            private String commandRequestId;
            private String name;

            private JSONArray params;
            @Data
                public static class IntentInfo {
                    private String domain;
                    private String intent;
                }
        }

    static int enterListeningTimes = 0;
    static State.DialogState currentState;
    static MultiModalDialog conversation;
    static int sampleRate = 16000;
    static String model = "multimodal-dialog";
    static String workspaceId = "your_workspace_id"; // 替换为您的Workspace ID
    static String appId = "your_app_id"; // 替换为您的 APP ID
    static String dialogId = "";
    static String apiKey = "your_api_key"; // 替换为您的API Key
    static String localRecordedFileUrl = ""; // 你的本地录音文件的fileUrl
    @Test
    public void testMultimodalDuplex() throws Exception {
        Constants.baseWebsocketApiUrl = "wss://dashscope.aliyuncs.com/api-ws/v1/inference";
        log.info("baseWebsocketApiUrl: {}", Constants.baseWebsocketApiUrl);

        /*
step1. 设置duplex
step2. 从本地麦进行录制，执行相关指令(至少依次说出， 开启录音/结束录音 两条指令)
step3. 每条指令收到回复后在callback中更新端侧状态
step4. 结束录用后用户需要手动提交转写，而后在callback中获取纪要转写结果
*/
        MultiModalRequestParam params =
            MultiModalRequestParam.builder()
            .customInput(MultiModalRequestParam.CustomInput.builder()
                         .appId(appId)
                         .workspaceId(workspaceId)
                         .dialogId("")
                         .build()
                        )
            .upStream(
                MultiModalRequestParam.UpStream.builder()
                .mode("duplex")
                .audioFormat("pcm")
                .build())
            .downStream(
                MultiModalRequestParam.DownStream.builder()
                .voice("longxiaochun_v2")
                .sampleRate(sampleRate)
                .build())
            .clientInfo(
                MultiModalRequestParam.ClientInfo.builder()
                .userId("1234")
                .device(MultiModalRequestParam.ClientInfo.Device.builder().uuid("1234").build())
                .build())
            .apiKey(apiKey)
            .model(model)
            .build();

        log.info("params: {}", JsonUtils.toJson(params));

        conversation = new MultiModalDialog(params, getCallback());

        conversation.start();
        while (currentState != State.DialogState.LISTENING) {
            try {
                sleep(100);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }

        // 发送音频
        sendAudioFromLocalMicrophone(conversation);

        conversation.stop();

        System.out.println("############ End Test duplex ############");
    }

    public void sendAudioFromLocalMicrophone(MultiModalDialog conversation) throws Exception {
        AudioFormat format = new AudioFormat(sampleRate, 16, 1, true, false);
        DataLine.Info info = new DataLine.Info(TargetDataLine.class, format);
        TargetDataLine targetLine = (TargetDataLine) AudioSystem.getLine(info);
        targetLine.open(format);
        targetLine.start();

        byte[] buffer = new byte[3200]; // 和服务端协议 buffer 匹配
        System.out.println("Start capture audio from microphone, press Ctrl+C to stop...");
        long start = System.currentTimeMillis();

        int i = 0;
        while (true) {
            int numBytesRead = targetLine.read(buffer, 0, buffer.length);
            if (numBytesRead > 0) {
                conversation.sendAudioData((ByteBuffer.wrap(buffer, 0, numBytesRead)));
            }
            // 1 分钟后自动停止发送
            if(System.currentTimeMillis() - start > 60000) { // 连续发送1分钟
                break;
            }
            ++i;
            if (i%10 == 0) {
                System.out.println("local microphone sending!");
            }
            Thread.sleep(100); // 控制发送速率
        }
        targetLine.stop();
        targetLine.close();
        System.out.println("Mic audio capture ended");
    }

    public static MultiModalDialogCallback getCallback() {
        return new MultiModalDialogCallbackImpl();
    }

    public static class MultiModalDialogCallbackImpl extends MultiModalDialogCallback {
        @Override
        public void onConnected() {}

        @Override
        public void onStarted(String dialogId) {

            log.info("onStarted: {}, self {}", dialogId, this);

        }

        private void changeToNewRecordingStatus(String newStatus) {

            JSONObject tingwuMeetingObj = new JSONObject();
            JSONObject clientRecordingStatus = new JSONObject();
            clientRecordingStatus.put("clientRecordingStatus", newStatus);
            tingwuMeetingObj.put("tingwu_meeting", clientRecordingStatus);

            // 调用updateInfo更新端侧录音状态到录音中
            conversation.updateInfo(
                MultiModalRequestParam.UpdateParams.builder()
                .bizParams(MultiModalRequestParam.BizParams.builder()
                           .userDefinedParams(
                               tingwuMeetingObj
                           )
                           .build())
                .build()
            );
        }

        @Override
        public void onStopped(String dialogId) {
            log.info("onStopped: {}", dialogId);
        }

        @Override
        public void onSpeechStarted(String dialogId) {
            log.info("onSpeechStarted: {}", dialogId);
        }

        @Override
        public void onSpeechEnded(String dialogId) {
            log.info("onSpeechEnded: {}", dialogId);
        }
        @Override
        public void onError(String dialogId, String errorCode, String errorMsg) {
            log.error("onError: {}, {}, {}, self {}", dialogId, errorCode, errorMsg, this);
            enterListeningTimes++ ; //force quit dialog test
        }

        @Override
        public void onStateChanged(State.DialogState state) {
            log.info("onStateChanged: {}", state);
            currentState = state;
            if (currentState == State.DialogState.LISTENING) {
                enterListeningTimes++;
                log.info("enterListeningTimes: {}", enterListeningTimes);
            }
        }

        @Override
        public void onSpeechAudioData(ByteBuffer audioData) {
            System.out.println("Received binary message (audio data)");
        }

        @Override
        public void onRespondingStarted(String dialogId) {
            log.info("onRespondingStarted: {}", dialogId);
            conversation.localRespondingStarted();
        }

        @Override
        public void onRespondingEnded(String s, JsonObject jsonObject) {
        }


        @Override
        public void onRespondingContent(String dialogId, JsonObject content) {
            log.info("onRespondingContent: {}, {}", dialogId, content);

            if (content != null && content.has("extra_info")) {
                JsonObject extraInfo = content.get("extra_info").getAsJsonObject();
                if (extraInfo != null && extraInfo.has("commands")) {
                    String commandsStr = extraInfo.get("commands").getAsString();
                    List<CommandDTO> commands = JSONObject.parseArray(commandsStr, CommandDTO.class);
                    for (CommandDTO commandDTO : commands) {
                        // 处理开启录音
                        if (commandDTO != null && commandDTO.getName().equals("start_local_recording")) {
                            changeToNewRecordingStatus("1");
                        }
                            // 处理暂停录音
                        else if (commandDTO != null && commandDTO.getName().equals("pause_local_recording")) {
                            changeToNewRecordingStatus("2");
                        }
                            // 处理恢复录音
                        else if (commandDTO != null && commandDTO.getName().equals("resume_local_recording")) {
                            changeToNewRecordingStatus("1");
                        }
                            // 处理结束录音
                        else if (commandDTO != null && commandDTO.getName().equals("end_local_recording")) {
                            changeToNewRecordingStatus("0");
                            System.out.println("Received end_local_recording command!");

                            String reqId = commandDTO.getCommandRequestId();
                            JSONObject invokeResObj = new JSONObject();
                            invokeResObj.put("fileUrl", localRecordedFileUrl);
                            JSONObject cmdRes = new JSONObject();
                            cmdRes.put("command_request_id", reqId);
                            cmdRes.put("invoke_result", JSONObject.toJSONString(invokeResObj));
                            JSONArray cmdResList = new JSONArray();
                            cmdResList.add(cmdRes);

                            Map<String, Object> paasThroughParams = new HashMap<>();
                            paasThroughParams.put("command_results", JSONObject.toJSON(cmdResList));

                            // 发送结束录音指令执行结果
                            conversation.requestToRespond(
                                "prompt",
                                "",
                                MultiModalRequestParam.UpdateParams.builder()
                                .bizParams(MultiModalRequestParam.BizParams.builder()
                                           .passThroughParams(paasThroughParams)
                                           .build())
                                .build()
                            );
                            try {
                                Thread.sleep(100);
                            } catch (InterruptedException e) {
                                throw new RuntimeException(e);
                            }
                        }
                            // 处理结束录音提交文件进行转写的返回结果
                        else if (commandDTO != null && commandDTO.getName().equals("end_local_recording_execution_res")) {
                            System.out.println("Received end_local_recording_execution_res command!");
                            System.out.println("dataId param is: " + JSONObject.toJSONString(commandDTO.getParams()));
                        }
                    }
                }
            }
        }

        @Override
        public void onSpeechContent(String dialogId, JsonObject content) {
            log.info("onSpeechContent: {}, {}", dialogId, content);
        }

        @Override
        public void onRequestAccepted(String dialogId) {
            log.info("onRequestAccepted: {}", dialogId);
        }

        @Override
        public void onClosed() {
            log.info("onClosed , self {}", this);
            enterListeningTimes++ ;
        }
    }
}