C++ SDK

#include <errno.h>
#include <pthread.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <ctime>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>

#include "nlsClient.h"
#include "nlsEvent.h"
#include "nlsToken.h"
#include "flowingSynthesizerRequest.h"

#define SAMPLE_RATE_16K 16000
#define DEFAULT_STRING_LEN 512
#define AUDIO_TEXT_LENGTH 2048

/**
 * Maintain a global service authentication token and its expiration timestamp.
 * Before each service call, check if the token has expired.
 * If it has, regenerate a token using the AccessKey ID and AccessKey
 * secret, and update the global token and its expiration timestamp.
 *
 * Note: Do not regenerate a new token before every service call. Regenerate it only when it is about to expire. All concurrent service calls can share one token.
 */
// Custom thread parameters
struct ParamStruct {
  char text[AUDIO_TEXT_LENGTH];
  char token[DEFAULT_STRING_LEN];
  char appkey[DEFAULT_STRING_LEN];
  char url[DEFAULT_STRING_LEN];

  pthread_mutex_t mtx;
};

// Custom callback parameters
struct ParamCallBack {
 public:
  explicit ParamCallBack(ParamStruct* param) {
    tParam = param;
    pthread_mutex_init(&mtxWord, NULL);
    pthread_cond_init(&cvWord, NULL);
  };
  ~ParamCallBack() {
    tParam = NULL;
    pthread_mutex_destroy(&mtxWord);
    pthread_cond_destroy(&cvWord);
  };

  pthread_mutex_t mtxWord;
  pthread_cond_t cvWord;

  ParamStruct* tParam;
};

std::string g_appkey = "";
std::string g_akId = "";
std::string g_akSecret = "";
std::string g_token = "";
std::string g_url = "";
std::string g_voice = "xiaoyun";
int g_threads = 1;
bool g_save_audio = true;
std::string g_text = "";
std::string g_format = "wav";
long g_expireTime = -1;
static int sample_rate = SAMPLE_RATE_16K;
static bool enableSubtitle = false;

/**
 * Regenerate a token and get its expiration timestamp based on the AccessKey ID and AccessKey secret
 */
int generateToken(std::string akId, std::string akSecret, std::string* token,
                  long* expireTime) {
  AlibabaNlsCommon::NlsToken nlsTokenRequest;
  nlsTokenRequest.setAccessKeyId(akId);
  nlsTokenRequest.setKeySecret(akSecret);

  int retCode = nlsTokenRequest.applyNlsToken();
  /* Get the failure cause */
  if (retCode < 0) {
    std::cout << "Failed error code: " << retCode
              << "  error msg: " << nlsTokenRequest.getErrorMsg() << std::endl;
    return retCode;
  }

  *token = nlsTokenRequest.getToken();
  *expireTime = nlsTokenRequest.getExpireTime();

  return 0;
}

void OnSynthesisStarted(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::cout << "OnSynthesisStarted:"
            << "  status code: " << cbEvent->getStatusCode()
            << "  task id: " << cbEvent->getTaskId()
            << "  all response:" << cbEvent->getAllResponse() << std::endl;
}

/**
 * @brief When the SDK receives a synthesis completion message from the cloud, the SDK's internal thread reports a Completed event.
 * @note After the Completed event is reported, the SDK closes the recognition channel.
 * @param cbEvent Callback event structure. For more information, see nlsEvent.h.
 * @param cbParam Custom callback parameters. The default is NULL. You can customize parameters as needed.
 * @return
 */
void OnSynthesisCompleted(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::cout
      << "OnSynthesisCompleted: "
      << ", status code: "
      << cbEvent
             ->getStatusCode()  // Get the status code of the message. 0 or 20000000 indicates success. A failure corresponds to a specific error code.
      << ", task id: "
      << cbEvent->getTaskId()  // The task ID of the current task. We recommend that you print this for troubleshooting.
      << std::endl;
  std::cout << "OnSynthesisCompleted: All response:"
            << cbEvent->getAllResponse()
            << std::endl;  // Get the complete information returned by the server.
}

/**
 * @brief If an exception occurs during synthesis, the SDK's internal thread reports a TaskFailed event.
 * @note After the TaskFailed event is reported, the SDK closes the recognition channel.
 * @param cbEvent Callback event structure. For more information, see nlsEvent.h.
 * @param cbParam Custom callback parameters. The default is NULL. You can customize parameters as needed.
 * @return
 */
void OnSynthesisTaskFailed(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  FILE* failed_stream = fopen("synthesisTaskFailed.log", "a+");
  if (failed_stream) {
    char outbuf[1024] = {0};
    snprintf(outbuf, sizeof(outbuf),
             "OnSynthesisTaskFailed status code:%d task id:%s error mesg:%s\n",
             cbEvent->getStatusCode(), cbEvent->getTaskId(),
             cbEvent->getErrorMessage());
    std::cout << outbuf << std::endl;
    fwrite(outbuf, strlen(outbuf), 1, failed_stream);
    fclose(failed_stream);
  }
}

/**
 * @brief When recognition ends or an exception occurs, the connection channel is closed,
 * and the SDK's internal thread reports a ChannelClosed event.
 * @param cbEvent Callback event structure. For more information, see nlsEvent.h.
 * @param cbParam Custom callback parameters. The default is NULL. You can customize parameters as needed.
 * @return
 */
void OnSynthesisChannelClosed(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  ParamCallBack* tmpParam = static_cast<ParamCallBack*>(cbParam);
  if (tmpParam) {
    std::cout << "OnSynthesisChannelClosed: "
              << ", All response: " << cbEvent->getAllResponse()
              << std::endl;  // Get the complete information returned by the server.
  }
}

/**
 * @brief After the text is sent to the server, the SDK receives binary audio data from the server.
 * The SDK's internal thread reports this to the user through a BinaryDataRecved event.
 * @param cbEvent Callback event structure. For more information, see nlsEvent.h.
 * @param cbParam Custom callback parameters. The default is NULL. You can customize parameters as needed.
 * @return
 * @notice Note: Do not perform blocking operations here. Only transfer the audio data. Performing too many operations in this callback
 *         will block subsequent data callbacks and the completed event callback.
 */
void OnBinaryDataRecved(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::vector<unsigned char> data =
      cbEvent->getBinaryData();  // getBinaryData() gets the binary audio data of the synthesized text.
  std::cout
      << "  OnBinaryDataRecved: status code: "
      << cbEvent
             ->getStatusCode()  // Get the status code of the message. 0 or 20000000 indicates success. A failure corresponds to a specific error code.
      << ", taskId: "
      << cbEvent->getTaskId()  // The task ID of the current task. We recommend that you print this for troubleshooting.
      << ", data size: " << data.size()  // The size of the data.
      << std::endl;

  if (g_save_audio && data.size() > 0) {
    // Append the binary audio data to a file.
    std::string dir = "./tts_audio";
    if (access(dir.c_str(), 0) == -1) {
      mkdir(dir.c_str(), S_IRWXU);
    }
    char file_name[256] = {0};
    snprintf(file_name, 256, "%s/%s.%s", dir.c_str(), cbEvent->getTaskId(),
             g_format.c_str());
    FILE* tts_stream = fopen(file_name, "a+");
    if (tts_stream) {
      fwrite((char*)&data[0], data.size(), 1, tts_stream);
      fclose(tts_stream);
    }
  }
}

void OnSentenceBegin(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::cout
      << "OnSentenceBegin "
      << "Response: "
      << cbEvent
             ->getAllResponse()  // Get the status code of the message. 0 or 20000000 indicates success. A failure corresponds to a specific error code.
      << std::endl;
}

void OnSentenceEnd(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::cout
      << "OnSentenceEnd "
      << "Response: "
      << cbEvent
             ->getAllResponse()  // Get the status code of the message. 0 or 20000000 indicates success. A failure corresponds to a specific error code.
      << std::endl;
}

/**
 * @brief Returns log information corresponding to the TTS text and incrementally returns the corresponding caption information.
 * @param cbEvent Callback event structure. For more information, see nlsEvent.h.
 * @param cbParam Custom callback parameters. The default is NULL. You can customize parameters as needed.
 * @return
 */
void OnSentenceSynthesis(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::cout
      << "OnSentenceSynthesis "
      << "Response: "
      << cbEvent
             ->getAllResponse()  // Get the status code of the message. 0 or 20000000 indicates success. A failure corresponds to a specific error code.
      << std::endl;
}

/**
 * @brief All information returned by the server is provided through this callback.
 * @param cbEvent Callback event structure. For more information, see nlsEvent.h.
 * @param cbParam Custom callback parameters. The default is NULL. You can customize parameters as needed.
 * @return
 */
void onMessage(AlibabaNls::NlsEvent* cbEvent, void* cbParam) {
  std::cout << "onMessage: All response:" << cbEvent->getAllResponse()
            << std::endl;
}

/**
 * @brief Worker thread in short-connection mode.
 *        The process loops as follows:
 *        createFlowingSynthesizerRequest   <----|
 *                   |                              |
 *           request->setSingleRoundText(text to merge)  |
 *                   |                              |
 *           request->start()                       |
 *                   |                              |
 *           Receive OnSynthesisChannelClosed callback |
 *                   |                              |
 *      releaseFlowingSynthesizerRequest(request) --|
 */
void* pthreadSingleRoundFunc(void* arg) {
  int testCount = 0;  // Counts the number of runs to exit after the configured number of loops.
  bool timedwait_flag = false;

  // Get the token, configuration files, and other parameters from the custom thread parameters.
  ParamStruct* tst = static_cast<ParamStruct*>(arg);
  if (tst == NULL) {
    std::cout << "arg is not valid." << std::endl;
    return NULL;
  }

  pthread_mutex_init(&(tst->mtx), NULL);

  // Initialize custom callback parameters.
  ParamCallBack cbParam(tst);

  /*
   * 1. Create a FlowingSynthesizerRequest object for long-text-to-speech synthesis.
   *
   * For more information about long-text-to-speech synthesis, see:
   * https://help.aliyun.com/zh/isi/developer-reference/streaming-text-to-speech-synthesis/
   */

  AlibabaNls::FlowingSynthesizerRequest* request =
      AlibabaNls::NlsClient::getInstance()->createFlowingSynthesizerRequest();
  if (request == NULL) {
    std::cout << "createFlowingSynthesizerRequest failed." << std::endl;
    return NULL;
  }

  /*
   * 2. Set the callbacks for receiving results.
   */
  // Set the callback function for when audio synthesis can start.
  request->setOnSynthesisStarted(OnSynthesisStarted, &cbParam);
  // Set the callback function for when audio synthesis is completed.
  request->setOnSynthesisCompleted(OnSynthesisCompleted, &cbParam);
  // Set the callback function for when the audio synthesis channel is closed.
  request->setOnChannelClosed(OnSynthesisChannelClosed, &cbParam);
  // Set the callback function for exceptions and failures.
  request->setOnTaskFailed(OnSynthesisTaskFailed, &cbParam);
  // Set the callback function for receiving text and audio data.
  request->setOnBinaryDataReceived(OnBinaryDataRecved, &cbParam);
  // Set the caption information.
  request->setOnSentenceSynthesis(OnSentenceSynthesis, &cbParam);
  // Beginning of a sentence.
  request->setOnSentenceBegin(OnSentenceBegin, &cbParam);
  // End of a sentence.
  request->setOnSentenceEnd(OnSentenceEnd, &cbParam);
  // Set the callback function for all messages returned by the server.
  // request->setOnMessage(onMessage, &cbParam);
  // Enable the callback function for all messages returned by the server. Other callbacks (except OnBinaryDataRecved) will be disabled.
  // request->setEnableOnMessage(true);

  /*
   * 3. Set the relevant parameters for the request.
   */
  // Voice, including "xiaoyun", "ruoxi", "xiaogang", etc. Optional. The default is xiaoyun.
  request->setVoice(g_voice.c_str());
  // Volume, range: 0-100. Optional. The default is 50.
  request->setVolume(50);
  // Audio encoding format. Optional. The default is wav. Supported formats: pcm, wav, mp3.
  request->setFormat("wav");
  // Audio sampling rate, including 8000 and 16000. Optional. The default is 16000.
  request->setSampleRate(sample_rate);
  // Speech rate, range: -500 to 500. Optional. The default is 0.
  request->setSpeechRate(0);
  // Pitch, range: -500 to 500. Optional. The default is 0.
  request->setPitchRate(0);
  // Enable captions.
  request->setEnableSubtitle(enableSubtitle);

  // Set the AppKey. Required. Apply for it on the official website.
  if (strlen(tst->appkey) > 0) {
    request->setAppKey(tst->appkey);
  }
  // Set the account verification token. Required.
  if (strlen(tst->token) > 0) {
    request->setToken(tst->token);
  }

  if (strlen(tst->url) > 0) {
    request->setUrl(tst->url);
  }
    
  // Set the connection timeout to 500 ms.
  // request->setTimeout(500);
  // Get the encoding format of the returned text.
  // const char* output_format = request->getOutputFormat();
  // std::cout << "text format: " << output_format << std::endl;

  /*
   * 4.
   * Set the text to be synthesized for long-text-to-speech synthesis.
   */
  if (request->setSingleRoundText(tst->text) != 0) {
    AlibabaNls::NlsClient::getInstance()->releaseFlowingSynthesizerRequest(
        request);  // If start() fails, release the request object.
    return NULL;
  }
    
  /*
   * 5.
   * start() is an asynchronous operation. If successful, it starts returning BinaryRecv events. If it fails, it returns a TaskFailed event.
   */
  std::cout << "start -> pid " << pthread_self() << std::endl;
  struct timespec outtime;
  struct timeval now;
  int ret = request->start();
  testCount++;
  if (ret < 0) {
    std::cout << "start failed. pid:" << pthread_self() << std::endl;
    const char* request_info = request->dumpAllInfo();
    if (request_info) {
      std::cout << "  all info: " << request_info << std::endl;
    }
    AlibabaNls::NlsClient::getInstance()->releaseFlowingSynthesizerRequest(
        request);  // If start() fails, release the request object.
    return NULL;
  } else {
    std::cout << "start success. pid " << pthread_self() << std::endl;
    /*
     * Wait for the started event to return, which indicates that start() was successful, and then send the audio data.
     * The voice server might not be able to process the current request in time, causing no callback to be returned within 10 seconds.
     * After 10 seconds, a TaskFailed callback is returned. Therefore, a timeout mechanism is required.
     */
    std::cout << "wait started callback." << std::endl;
    gettimeofday(&now, NULL);
    outtime.tv_sec = now.tv_sec + 10;
    outtime.tv_nsec = now.tv_usec * 1000;
    pthread_mutex_lock(&(cbParam.mtxWord));
    if (ETIMEDOUT == pthread_cond_timedwait(&(cbParam.cvWord),
                                            &(cbParam.mtxWord), &outtime)) {
      std::cout << "start timeout" << std::endl;
      timedwait_flag = true;
      pthread_mutex_unlock(&(cbParam.mtxWord));
      // If the start() call times out, cancel() cancels the current request.
      request->cancel();
      return NULL;
    }
    pthread_mutex_unlock(&(cbParam.mtxWord));
  }

  /*
   * 6. If start() is successful, wait to receive all synthesized data.
   *    The stop() interface is not meaningful here. The process will run to completion whether it is called or not.
   *    cancel() stops the task immediately and does not return a callback. A TaskFailed event is returned on failure.
   */
  //    ret = request->cancel();
  ret = request->stop();

  /*
   * Start waiting to receive all synthesized data.
   */
  if (ret == 0) {
    /*
     * Wait for the started event to return, which indicates that start() was successful, and then send the audio data.
     * The voice server might not be able to process the current request in time, causing no callback to be returned within 10 seconds.
     * After 10 seconds, a TaskFailed callback is returned. Therefore, a timeout mechanism is required.
     */
    // Wait for the closed event before releasing, otherwise a crash may occur.
    std::cout << "wait closed callback." << std::endl;
    /*
     * The voice server might not be able to process the current request in time, causing no callback to be returned within 10 seconds.
     * After 10 seconds, a TaskFailed callback is returned with the error message:
     * "Gateway:IDLE_TIMEOUT:Websocket session is idle for too long time,
     * the last directive is 'XXXX'!" Therefore, a timeout mechanism is required.
     */
    gettimeofday(&now, NULL);
    outtime.tv_sec = now.tv_sec + 30;
    outtime.tv_nsec = now.tv_usec * 1000;
    // Wait for the closed event before releasing, otherwise a crash may occur.
    pthread_mutex_lock(&(cbParam.mtxWord));
    if (ETIMEDOUT == pthread_cond_timedwait(&(cbParam.cvWord),
                                            &(cbParam.mtxWord), &outtime)) {
      std::cout << "stop timeout" << std::endl;
      pthread_mutex_unlock(&(cbParam.mtxWord));
      return NULL;
    }
    pthread_mutex_unlock(&(cbParam.mtxWord));
  } else {
    std::cout << "ret is " << ret << ", pid " << pthread_self() << std::endl;
  }
  gettimeofday(&now, NULL);
  std::cout << "current request task_id:" << request->getTaskId() << std::endl;
  std::cout << "stop finished. pid " << pthread_self() << " tv: " << now.tv_sec
            << std::endl;

  /*
   * 7. After all tasks are complete, release the current request.
   *    Release the request only after the closed event (which confirms all work is done) to avoid disrupting the internal state machine,
   * which could forcibly uninstall a running request.
   */
  const char* request_info = request->dumpAllInfo();
  if (request_info) {
    std::cout << "  all info: " << request_info << std::endl;
  }
  AlibabaNls::NlsClient::getInstance()->releaseFlowingSynthesizerRequest(
      request);
  std::cout << "release Synthesizer success. pid " << pthread_self()
            << std::endl;

  pthread_mutex_destroy(&(tst->mtx));

  return NULL;
}

/**
 * Synthesize multiple text data.
 * In the SDK, multi-threading means one thread per text data, not multiple threads for one text data.
 * The example code starts four threads to synthesize four files simultaneously.
 * Free-tier users cannot have more than two concurrent connections.
 */
#define AUDIO_TEXT_NUMS 4
#define AUDIO_FILE_NAME_LENGTH 32
int flowingSynthesizerMultFile(const char* appkey, int threads) {
  /**
   * Get the current system timestamp to check if the token has expired.
   */
  std::time_t curTime = std::time(0);
  if (g_token.empty()) {
    if (g_expireTime - curTime < 10) {
      std::cout << "the token will be expired, please generate new token by "
                   "AccessKey-ID and AccessKey-Secret."
                << std::endl;
      int ret = generateToken(g_akId, g_akSecret, &g_token, &g_expireTime);
      if (ret < 0) {
        std::cout << "generate token failed" << std::endl;
        return -1;
      } else {
        if (g_token.empty() || g_expireTime < 0) {
          std::cout << "generate empty token" << std::endl;
          return -2;
        }
        std::cout << "token: " << g_token << std::endl;
      }
    }
  }

  /* Do not exceed AUDIO_TEXT_LENGTH */
  const char texts[AUDIO_TEXT_LENGTH] = {
      "This is a long text for speech synthesis. It demonstrates the capability of the streaming and long-text-to-speech synthesis feature of Alibaba Cloud Intelligent Speech Interaction. The quick brown fox jumps over the lazy dog."};

  ParamStruct pa[threads];

  for (int i = 0; i < threads; i++) {
    memset(pa[i].token, 0, DEFAULT_STRING_LEN);
    memcpy(pa[i].token, g_token.c_str(), g_token.length());

    memset(pa[i].appkey, 0, DEFAULT_STRING_LEN);
    memcpy(pa[i].appkey, appkey, strlen(appkey));

    memset(pa[i].text, 0, AUDIO_TEXT_LENGTH);
    if (g_text.empty()) {
      memcpy(pa[i].text, texts, strlen(texts));
    } else {
      memcpy(pa[i].text, g_text.data(), strlen(g_text.data()));
    }

    memset(pa[i].url, 0, DEFAULT_STRING_LEN);
    if (!g_url.empty()) {
      memcpy(pa[i].url, g_url.c_str(), g_url.length());
    }
  }

  std::vector<pthread_t> pthreadId(threads);
  for (int j = 0; j < threads; j++) {
    pthread_create(&pthreadId[j], NULL, &pthreadSingleRoundFunc,
                   (void*)&(pa[j]));
  }

  std::cout << "start pthread_join..." << std::endl;

  for (int j = 0; j < threads; j++) {
    pthread_join(pthreadId[j], NULL);
  }

  std::cout << "flowingSynthesizerMultFile exit..." << std::endl;
  return 0;
}

int main(int argc, char* argv[]) {
  std::string g_appkey = getenv("NLS_APPKEY_ENV");
  g_akId = getenv("NLS_AK_ENV");
  g_akSecret = getenv("NLS_SK_ENV");

  std::cout << " appKey: " << g_appkey << std::endl;
  std::cout << " akId: " << g_akId << std::endl;
  std::cout << " akSecret: " << g_akSecret << std::endl;
  std::cout << " voice: " << g_voice << std::endl;
  std::cout << "\n" << std::endl;

  int ret = AlibabaNls::NlsClient::getInstance()->setLogConfig(
      "log-flowingSynthesizer", AlibabaNls::LogDebug, 400, 50, NULL);
  if (ret < 0) {
    std::cout << "set log failed." << std::endl;
    return -1;
  }

  // Set the socket address type required by the operating environment. The default is AF_INET.
  // This must be called before startWorkThread().
  // AlibabaNls::NlsClient::getInstance()->setAddrInFamily("AF_INET");

  // Set the direct connection IP in a private cloud deployment.
  // This must be called before startWorkThread().
  // AlibabaNls::NlsClient::getInstance()->setDirectHost("xxx.xxx.xxx.xxx");

  std::cout << "startWorkThread begin... " << std::endl;

  // Start the worker thread. This function must be called before creating and starting a request.
  // If the input parameter is negative, the number of available cores in the current system is started.
  // For high concurrency, 4 is recommended. For a single request, 1 is recommended.
  // If CPU usage is high with high concurrency, you can enter -1 to enable all CPU cores.
  AlibabaNls::NlsClient::getInstance()->startWorkThread(1);

  std::cout << "startWorkThread finish" << std::endl;

  // Synthesize multiple texts.
  ret = flowingSynthesizerMultFile(g_appkey.c_str(), g_threads);
  if (ret) {
    std::cout << "flowingSynthesizerMultFile failed." << std::endl;
    AlibabaNls::NlsClient::releaseInstance();
    return -2;
  }

  // After all work is done, release nlsClient before the process exits.
  // Note that releaseInstance() is not thread-safe.
  std::cout << "releaseInstance -> " << std::endl;
  AlibabaNls::NlsClient::releaseInstance();
  std::cout << "releaseInstance done." << std::endl;

  return 0;
}