RTOS SDK (license mode): Vision module integration

2.3.1. Visual question answering and photo translation

The following code shows an example of how to handle events for visual question answering and photo translation:

static int32_t _visual_callback(uint32_t event, void* param)
{
    (void)param;
    switch (event) {
    case C_VISUAL_EVENT_VQA_START:
        UTIL_LOG_I("vqa start");
        c_camera_open();
        // Manually trigger a photo capture. Whether to call this function depends on your specific implementation.
        c_camera_capture();
        break; 
    case C_VISUAL_EVENT_VQA_END:
        UTIL_LOG_I("vqa end");
        c_camera_close();
        break;
    ...
    }
    return UTIL_SUCCESS;
}

2.3.2. Video call and instant video call

The following code shows an example of how to handle events for video calls and instant video calls:

static int32_t _visual_callback(uint32_t event, void* param)
{
    (void)param;
    switch (event) {
    case C_VISUAL_EVENT_LIVEAI_START:
        UTIL_LOG_I("liveai start");
        c_camera_open();
        break;
    case C_VISUAL_EVENT_LIVEAI_ACTION:
        UTIL_LOG_I("liveai capture");
        // Manually trigger a photo capture. Whether to call this function depends on your specific implementation.
        c_camera_capture();
        break;
    case C_VISUAL_EVENT_LIVEAI_STOP:
        UTIL_LOG_I("liveai stop");
        c_camera_close();
        break;
    ...
    }
    return UTIL_SUCCESS;
}

2.4.1. Image capture logic

Use the following function sequence to pass the captured image to the c_visual module. The module then encapsulates and uploads the data as required by the vision model.

Sample Code 1: Passing captured data to c_visual via a camera callback.

// Pass the image data captured by the camera to c_visual
int32_t hal_camera_capture_callback(uint8_t *image, uint32_t image_size)
{
  uint8_t *buffer;
  uint32_t buffer_size;
  buffer = c_visual_image_get_buffer(&buffer_size);
  if (buffer == NULL) {
    UTIL_LOG_W("get buffer fail");
    return -1;
  }
  if (buffer_size < image_size) {
    UTIL_LOG_W("image_siz > buffer_size");
    return -1;
  }
  memcpy(buffer, image, image_size);
  c_visual_image_action(image_size);
  return 0;
}

Sample Code 2: Creating a thread to periodically send camera data.

void hal_camera_task_handle(void *param)
{
  (void)params;
  int32_t err;
  uint8_t *buffer;
  uint32_t buffer_size;
  while (1) {
    buffer = c_visual_image_get_buffer(&buffer_size);
    if (buffer == NULL) {
      UTIL_LOG_D("get buffer fail");
      util_msleep(20);
      continue;
    }
    err = hal_camera_capture(buffer, &buffer_size);
    if (err) {
      UTIL_LOG_D("capture fail");
      util_msleep(20);
      continue;
    }
    UTIL_LOG_I("capture success");
    c_visual_image_action(buffer_size);
    util_msleep(100);  // Sleep before the next capture.
  }
}

2.4.2. Implementation example

The following sample code creates a thread to periodically capture images from the camera. Choose an implementation method suitable for your hardware.

#include "hal_camera.h"
#include "c_visual.h"
#define CAMERA_DROP_NUM         (4)
typedef struct {
    uint8_t camera_initd;
    uint8_t camera_work;
    util_task_t* task;
    uint8_t task_running;
    uint8_t drop_img_num;
} _camera_info_t;
static _camera_info_t _camera_info = { 0 };
static void _camera_task_entry(void* params) {
    (void)params;
    int32_t err;
    uint8_t *buffer;
    uint32_t buffer_size;
    while (_camera_info.task_running) {
        if (_camera_info.camera_work == 0) {
            util_msleep(20);
            continue;
        }
        buffer = c_visual_image_get_buffer(&buffer_size);
        if (buffer == NULL) {
            UTIL_LOG_D("get buffer fail");
            util_msleep(20);
            continue;
        }
        err = hal_camera_capture(buffer, &buffer_size);
        if (err) {
            UTIL_LOG_D("capture fail");
            util_msleep(20);
            continue;
        }
        UTIL_LOG_I("capture success");
        if (_camera_info.drop_img_num) {
            UTIL_LOG_I("drop [%d]", _camera_info.drop_img_num);
            _camera_info.drop_img_num--;
            continue;
        }
        c_visual_image_action(buffer_size);
        c_visual_task_handle();
    }
    util_task_t* del_task = _camera_info.task;
    memset(&_camera_info, 0, sizeof(_camera_info));
    hal_camera_close();
    UTIL_LOG_I("task will exit");
    util_task_delete(del_task);
}
int32_t c_camera_init(void)
{
    int32_t err;
    if (_camera_info.camera_initd) {
        return UTIL_SUCCESS;
    }
    memset(&_camera_info, 0, sizeof(_camera_info));
    _camera_info.drop_img_num = CAMERA_DROP_NUM;
    hal_camera_open();
    _camera_info.task_running = 1;
    _camera_info.task = util_task_create("CAMERA", CAMERA_TASK_PRIORITY, CAMERA_STACK_SIZE, _camera_task_entry, NULL);
    _camera_info.camera_initd = 1;
    UTIL_LOG_I("done");
    return UTIL_SUCCESS;
}
int32_t c_camera_open(void)
{
    if (_camera_info.camera_initd == 0) {
        c_camera_init();
    }
    if (_camera_info.camera_work) {
        UTIL_LOG_D("already");
        return UTIL_SUCCESS;
    }
    c_visual_data_reset();
    hal_camera_open();
    _camera_info.drop_img_num = CAMERA_DROP_NUM;
    _camera_info.camera_work = 1;
    UTIL_LOG_I("done");
    return UTIL_SUCCESS;
}
int32_t c_camera_close(void)
{
    if (_camera_info.camera_initd == 0) {
        UTIL_LOG_E("not initialized");
        return UTIL_ERR_NO_INIT;
    }
    if (_camera_info.camera_work == 0) {
        UTIL_LOG_D("already");
        return UTIL_SUCCESS;
    }
    _camera_info.camera_work = 0;
    hal_camera_close();
    UTIL_LOG_I("done");
    return UTIL_SUCCESS;
}
int32_t c_camera_deinit(void)
{
    if (_camera_info.camera_initd == 0) {
        UTIL_LOG_D("already");
        return UTIL_SUCCESS;
    }
    _camera_info.task_running = 0;
    UTIL_LOG_I("prepare");
    return UTIL_SUCCESS;
}