WeChat mini program

Prerequisites

• Activate Intelligent Speech Interaction, create a project in the Intelligent Speech Interaction console, and obtain its Appkey.

• Install WeChat DevTools and configure a mini program project. WeChat Base Library 2.4.4 or later is required.

• Obtain permission to record audio in the mini program. In its server domain configuration, add the HTTPS domain of the application server that provides tokens to the allowed request domains, and add wss://nls-gateway-cn-shanghai.aliyuncs.com to the allowed socket domains.

Limitations

Each short sentence recognition request supports up to 60 seconds of audio. The audio format and sample rate must match the recognition parameters. The example uses mono PCM audio at 16 kHz and stops recording after 55 seconds.

Download and install

Download the SDK from the GitHub repository or download alibabacloud-nls-wx-sdk-master.zip directly. Copy the SDK's utils directory into the mini program project, then use require with the appropriate relative path to import a module.

Obtain a token

Obtain an NLS token on the application server and provide it to the mini program through an authenticated HTTPS endpoint. For the token acquisition procedure, see Obtain a token.

Keep the AccessKey ID and AccessKey secret on the server. Do not include them in mini program code or local storage. The server can cache a token based on its expiration time and obtain a new token before it expires to avoid frequent token requests.

Code example

The example records and streams audio after recognition starts. When recording stops, it waits for both the last audio frame and the recording stop event before requesting the final result. Application-side timeouts cover starting recognition, finishing recognition, and waiting for recording to stop. Adjust these timeouts to suit the application; they are not service limits.

Replace APPKEY and TOKEN_URL, and add the authentication required by the application server to the request in getToken(). TOKEN_URL is an application-defined endpoint, not an NLS OpenAPI endpoint. The example expects it to return token (a string) and expireTime (a Unix timestamp in seconds).

Register the pages/sr/sr page in the project's app.json and configure the recording purpose and privacy disclosures required by WeChat. The following JavaScript and WXML together implement the page.

Page logic: pages/sr/sr.js

const SpeechRecognition = require("../../utils/sr");
const APPKEY = "YOUR_APPKEY";
const TOKEN_URL = "https://your-server.example.com/nls/token";
const GATEWAY = "wss://nls-gateway-cn-shanghai.aliyuncs.com/ws/v1";

function withTimeout(promise, milliseconds) {
  let timer;
  return Promise.race([
    promise,
    new Promise((_, reject) => {
      timer = setTimeout(() => reject(new Error("Operation timed out")), milliseconds);
    })
  ]).finally(() => clearTimeout(timer));
}

function getToken() {
  return new Promise((resolve, reject) => {
    wx.request({
      url: TOKEN_URL,
      method: "GET",
      timeout: 10000,
      // Add the authentication required by your application server.
      success(res) {
        const value = res.data;
        if (res.statusCode === 200 && value && typeof value.token === "string"
            && value.token && Number(value.expireTime) > Date.now() / 1000 + 60) {
          resolve(value.token);
        } else {
          reject(new Error("No valid token returned"));
        }
      },
      fail: reject
    });
  });
}

Page({
  data: { busy: false, recording: false, result: "", error: "" },

  onLoad() {
    this.recorder = wx.getRecorderManager();
    const listen = (name, handler) => {
      this[name] = (...args) => {
        if (!this.unloaded) handler(...args);
      };
      this.recorder[name](this[name]);
    };
    listen("onStart", () => {
      if (!this.session) {
        this.recorder.stop();
        return;
      }
      this.setData({ recording: true });
    });
    listen("onFrameRecorded", (frame) => {
      const session = this.session;
      if (!session || !session.sr || session.closing) return;
      try {
        if (frame.frameBuffer.byteLength && !session.sr.sendAudio(frame.frameBuffer)) {
          throw new Error("Audio connection unavailable");
        }
        if (frame.isLastFrame) session.lastFrame = true;
        this.finishRecording(session);
      } catch (error) {
        this.abort(error.message);
      }
    });
    listen("onStop", () => {
      this.recorderBusy = false;
      const session = this.session;
      if (!session) return;
      session.stopped = true;
      this.setData({ recording: false });
      this.waitForLastFrame(session);
      this.finishRecording(session);
    });
    listen("onError", () => {
      this.recorderBusy = false;
      this.abort("Recording failed. Check microphone permission.");
    });
  },

  async onSrStart() {
    if (this.session || this.recorderBusy) return;
    const session = {};
    this.session = session;
    this.setData({ busy: true, recording: false, result: "", error: "" });
    try {
      const token = await getToken();
      if (this.session !== session) return;
      const sr = new SpeechRecognition({ url: GATEWAY, appkey: APPKEY, token });
      session.sr = sr;
      sr.on("changed", (message) => {
        if (this.session === session) {
          this.setData({ result: JSON.parse(message).payload.result || "" });
        }
      });
      sr.on("completed", (message) => {
        if (this.session !== session) return;
        session.completed = true;
        const response = JSON.parse(message);
        console.log("task_id:", response.header.task_id);
        this.setData({ result: response.payload.result || "" });
      });
      sr.on("failed", (message) => {
        if (this.session !== session) return;
        const header = JSON.parse(message).header;
        console.error("status:", header.status, "task_id:", header.task_id);
        this.abort("Recognition failed. Check the status code.");
      });
      sr.on("closed", () => {
        if (this.session === session && !session.completed) {
          this.abort("Connection closed before recognition completed.");
        }
      });

      const params = sr.defaultStartParams();
      params.enable_voice_detection = false;
      await withTimeout(sr.start(params), 10000);
      if (this.session !== session) return;
      this.recorderBusy = true;
      try {
        this.recorder.start({
        duration: 55000,
        numberOfChannels: 1,
        sampleRate: 16000,
        format: "PCM",
        frameSize: 4
        });
      } catch (error) {
        this.recorderBusy = false;
        throw error;
      }
    } catch (error) {
      if (this.session === session) this.abort(error.message || "Unable to start recognition.");
    }
  },

  onSrStop() {
    const session = this.session;
    if (!session || !this.data.recording || session.stopRequested) return;
    session.stopRequested = true;
    this.setData({ recording: false });
    this.waitForLastFrame(session);
    this.recorder.stop();
  },

  waitForLastFrame(session) {
    if (session.timer) return;
    session.timer = setTimeout(() => {
      if (this.session === session) this.abort("Timed out while stopping recording.");
    }, 5000);
  },

  async finishRecording(session) {
    if (!session.stopped || !session.lastFrame || session.closing) return;
    session.closing = true;
    clearTimeout(session.timer);
    try {
      await withTimeout(session.sr.close(), 10000);
      if (this.session === session) {
        this.session = null;
        this.setData({ busy: false, recording: false });
      }
    } catch (error) {
      if (this.session === session) this.abort(error.message || "Unable to finish recognition.");
    }
  },

  abort(message) {
    const session = this.session;
    this.session = null;
    if (session) {
      clearTimeout(session.timer);
      if (session.sr) session.sr.shutdown();
    }
    if (this.recorderBusy) this.recorder.stop();
    if (!this.unloaded) {
      this.setData({ busy: false, recording: false, error: message || "" });
    }
  },

  onHide() { this.abort(); },
  onUnload() {
    this.unloaded = true;
    this.abort();
    for (const name of ["onStart", "onFrameRecorded", "onStop", "onError"]) {
      const off = this.recorder["off" + name.slice(2)];
      if (typeof off === "function") off.call(this.recorder, this[name]);
    }
  }
});

Page layout: pages/sr/sr.wxml

<view>
  <button bindtap="onSrStart" disabled="{{busy}}">Start recognition</button>
  <button bindtap="onSrStop" disabled="{{!recording}}">Stop recognition</button>
  <text>{{result}}</text>
  <text>{{error}}</text>
</view>

result displays recognized text, and error displays error messages. For service errors, the example logs the status code and task ID to the console without logging the token.

API reference

{
  "format": "pcm",
  "sample_rate": 16000,
  "enable_intermediate_result": true,
  "enable_punctuation_prediction": true,
  "enable_inverse_text_normalization": true,
  "enable_voice_detection": true,
  "max_end_silence": 2000
}

Related information

• To continuously recognize longer speech, see the WeChat mini program SDK for real-time speech recognition.