Web-based stream ingest

Start recording

1. Enable recording

navigator.getUserMedia({
  audio: true
}, stream => {
  globalRecorder = new Recorder(stream);
  console.log('Start recording');
  connectWebSocket();
}, error => {
  console.log(error);
  // TODO: Handle exceptions and errors
})

2. Process the audio data format

   After the recording starts, you must merge and compress the raw audio stream. Then, process the stream to conform to the real-time audio stream format supported by the Tingwu API.

• Audio compression

compress: function () { // Merge and compress the data
  var data = new Float32Array(this.size);
  var offset = 0;
  for (var i = 0; i < this.buffer.length; i++) {
    data.set(this.buffer[i], offset);
    offset += this.buffer[i].length;
  }
  var compression = parseInt(this.inputSampleRate / this.outputSampleRate);
  var length = data.length / compression;
  var result = new Float32Array(length);
  var index = 0,
    j = 0;
  while (index < length) {
    result[index] = data[j];
    j += compression;
    index++;
  }
  return result;
},

• Convert the stream to the format required by the Tingwu API.

encodePCM: function () {
  var sampleRate = Math.min(this.inputSampleRate, 
     this.outputSampleRate);
  var sampleBits = Math.min(this.inputSampleBits, 
     this.outputSampleBits);
  var bytes = this.compress();
  var dataLength = bytes.length * (sampleBits / 8);
  var buffer = new ArrayBuffer(dataLength);
  var data = new DataView(buffer);
  var offset = 0;
  for (var i = 0; i < bytes.length; i++, offset += 2) {
    var s = Math.max(-1, Math.min(1, bytes[i]));
    data.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
  }
  return new Blob([data]);
}

Audio data transmission

1. Establish a WSS connection

function connectWebSocket() {
    ws = new WebSocket("Enter the MeetingJoinUrl returned when you created the real-time meeting"); 
    ws.binaryType = 'arraybuffer'; // Transmit data of the ArrayBuffer type
    ws.onopen = function () {
      if (ws.readyState == 1) {
        globalRecorder.start();
      }
      const obj = {
        header: {
          name: "StartTranscription", 
          namespace: "SpeechTranscriber", 
        },
        payload: {
          format: "pcm", // The example uses the pcm format. Select a format as needed. If you do not use the pcm format, the encodePCM method in this example is not applicable. You must implement your own data format transformation.
        }
        
      }
      ws.send(JSON.stringify(obj));
    };

    ws.onmessage = function (msg) {
      console.info(msg)
    }

    ws.onerror = function (err) {
      console.info(err)
    }
  }

2. Audio Transmission

var sendAudioData = function () { // Process data in chunks
      var reader = new FileReader();
      reader.onload = e => {
        var outbuffer = e.target.result;
        var arr = new Int8Array(outbuffer);
        if (arr.length > 0) {
          var tmparr = new Int8Array(1024);
          var j = 0;
          for (var i = 0; i < arr.byteLength; i++) {
            tmparr[j++] = arr[i];
            if (((i + 1) % 1024) == 0) {
              ws.send(tmparr);
              if (arr.byteLength - i - 1 >= 1024) {
                tmparr = new Int8Array(1024);
              } else {
                tmparr = new Int8Array(arr.byteLength - i - 1);
              }
              j = 0;
            }
            if ((i + 1 == arr.byteLength) && ((i + 1) % 1024) != 0) {
              ws.send(tmparr);
            }
          }
        }
      };
      reader.readAsArrayBuffer(audioData.encodePCM());
      audioData.clear();
    };

Callback processing

After the client sends audio data in a loop, you can continuously receive recognition and translation results based on the following returned events.

 ws.onmessage = function (msg) {
        console.info(msg);
        // msg.data is the received data (for details, see "Real-time stream ingest return events").
        // Process the data as needed.
        if (typeof msg.data === "string") {
          const dataJson = JSON.parse(msg.data);
          switch (dataJson.header.name) {
            case "SentenceBegin": {
              // SentenceBegin event
              console.log("Sentence", dataJson.payload.index, "starts");
              break;
            }
            case "TranscriptionResultChanged":
              // TranscriptionResultChanged event
              console.log(
                "Sentence " + dataJson.payload.index + " intermediate result:",
                dataJson.payload.result
              );
              break;

            case "SentenceEnd": {
              // SentenceEnd event
              console.log(
                "Sentence " + dataJson.payload.index + " ends:",
                dataJson.payload.result + dataJson.payload.stash_result.text
              );
              break;
            }
            case "ResultTranslated": {
              // ResultTranslated event
              console.log(
                "Sentence translation result",
                JSON.stringify(dataJson.payload.translate_result)
              );
              break;
            }
            //... 
          }
        }
      };

Stop recording

    const params = {
      header: {
        name: 'StopTranscription', 
        namespace: 'SpeechTranscriber',
      },
      payload: {},
    };
    ws.send(JSON.stringify(params));
    setTimeout(() => {
      ws.close();
    }, 10000);

Example result

The output of the preceding code example is shown in the following figure.