Connecting to models and applications

Overall sequence diagram

Create engine and set callbacks

let config = AoqCreateConfig()
config.workDir = workDir
config.enableDumpAudio = false
engine = AoqClientEngine.createEngine(config, delegate: self)

Implement the AoqEngineDelegate protocol to listen for callbacks such as onConnectionStatusChange, onDataMsg, and onError.

Start audio capture and playback

// Audio capture
let capCfg = AoqAudioCaptureConfig()
capCfg.channel = 1; capCfg.isExternal = false
engine.startAudioCapture(capCfg)

// Audio playback
let playCfg = AoqAudioPlaybackConfig()
playCfg.channel = 1; playCfg.isExternal = false
engine.startAudioPlayer(playCfg)

// Video capture (optional)
let vidCfg = AoqVideoCaptureConfig()
vidCfg.width = 720; vidCfg.height = 1280; vidCfg.fps = 15
engine.startVideoCapture(vidCfg)

Get connection credentials

The business AppServer proxies the request to Model Studio. See Token authentication.

Configure codecs and establish connection

Configure the codec parameters, then call connect:

// Audio codec configuration
let encCfg = AoqAudioCodecConfig()
encCfg.codecType = .audioPCM; encCfg.sampleRate = 16000; encCfg.channel = 1
engine.setAudioEncoderConfig(encCfg)
engine.setAudioDecoderConfig(encCfg)

// Disable media sending before connect; enable it after session.updated
engine.enableSendMediaStream(.audio, enable: false)

let config = AoqConnectConfig()
config.token = token
config.sid = sid
config.certFingerprint = certificate
config.relayEndpoints = relayEndpoints
config.workspaceIdHash = workspaceIdHash
config.publishTracks = [audioTrack, dataTrack]
config.subscribeTracks = [audioTrack, dataTrack]
engine.connect(config)

Configure AI session

After the connection succeeds, send a session.update event. For details, see Client events:

func onConnectionStatusChange(_ status: AoqConnectionStatus) {
    if status == .connected { sendSessionUpdate() }
}

private func sendSessionUpdate() {
    let json = """
    {
      // The ID of this event, generated by the client
      "event_id": "event_ToPZqeobitzUJnt3QqtWg",
      // Event type, fixed to session.update
      "type": "session.update",
      // Session configuration
      "session": {
          // Output modalities. Set to ["text"] (text only) or ["text","audio"] (text and audio).
          "modalities": [
              "text",
              "audio"
          ],
          // Voice for the output audio
          "voice": "Ethan",
          // Input audio format. Only pcm is supported. The input audio is a PCM audio stream with a 16 kHz sample rate.
          "input_audio_format": "pcm",
          // Output audio format. Only pcm is supported. The output audio is a PCM audio stream with a 24 kHz sample rate.
          "output_audio_format": "pcm",
          // System message that sets the model's goal or role.
          "instructions": "You are an AI customer service agent at a five-star hotel. Accurately and courteously answer customer questions about room types, facilities, prices, and booking policies. Always respond in a professional and helpful manner, and never provide unverified information or information beyond the scope of the hotel's services.",
          // Whether to enable voice activity detection. To enable it, pass a configuration object; the server then automatically detects when speech starts and stops.
          // Set to null to let the client decide when to trigger a model response.
          "turn_detection": {
              // VAD type: server_vad or semantic_vad. semantic_vad is recommended for qwen3.5-omni-realtime series models.
              "type": "semantic_vad",
              // VAD detection threshold. Increase it in noisy environments and decrease it in quiet environments.
              "threshold": 0.5,
              // Silence duration for detecting the end of speech. A model response is triggered after this duration is exceeded
              "silence_duration_ms": 800
          }
      }
    }
    """
    let msg = AoqDataMsg()
    msg.data = json.data(using: .utf8)!
    engine.send(msg)
}

Enable media sending after receiving session.updated

The following example handles the session.updated reply from the model. For details, see Server events:

func onDataMsg(_ msg: AoqDataMsg) {
    guard let obj = try? JSONSerialization.jsonObject(with: msg.data) as? [String: Any],
          let type = obj["type"] as? String else { return }
    if type == "session.updated" {
        engine.enableSendMediaStream(.audio, enable: true)
        engine.enableSendMediaStream(.video, enable: true)
    }
}

Disconnect and destroy engine

engine.disconnect()
AoqClientEngine.destroy()

Overall flow diagram

Establish connection

# pip install aiortc aiohttp certifi
import asyncio, aiohttp, ssl, certifi
from aiortc import RTCPeerConnection, RTCConfiguration, RTCSessionDescription
from aiortc.mediastreams import AudioStreamTrack

API_KEY = "your-api-key"
MODEL = "target-model"
SIGNALING_URL = f"https://{{endpoint}}/api/v1/webrtc/realtime?model={MODEL}"

async def connect():
    pc = RTCPeerConnection(RTCConfiguration(iceServers=[]))

    # Add an audio track so that the Offer SDP contains m=audio (required by the server)
    pc.addTrack(AudioStreamTrack())

    # Create a DataChannel to trigger SDP negotiation (the name is customizable; the server pushes events through a channel named "txt")
    pc.createDataChannel("oai-events")

    # SDP exchange: create an Offer and send it to the server
    offer = await pc.createOffer()
    await pc.setLocalDescription(offer)

    async with aiohttp.ClientSession() as session:
        async with session.post(
            SIGNALING_URL,
            ssl=ssl.create_default_context(cafile=certifi.where()),
            data=offer.sdp.encode("utf-8"),
            headers={
                "Content-Type": "application/sdp",
                "Authorization": f"Bearer {API_KEY}",
            },
        ) as resp:
            if not resp.ok:
                raise Exception(f"SDP exchange failed: {resp.status} {await resp.text()}")
            answer_sdp = await resp.text()

    print("=== Offer SDP ===")
    print(offer.sdp)
    print("=== Answer SDP ===")
    print(answer_sdp)

    # ICE connection setup completes automatically
    await pc.setRemoteDescription(RTCSessionDescription(sdp=answer_sdp, type="answer"))
    print("WebRTC connection established")
    return pc

Configure model parameters

Listen for messages that the model returns over the DataChannel to keep the interaction sequence correct:

pc.ondatachannel = (event) => {
  const ch = event.channel;
  ch.onmessage = (e) => {
    let obj;
    try { obj = JSON.parse(e.data); }
    catch (err) {
      return;
    }
    if (obj?.type === "session.created") {
      sendUpdate(event.channel);
      // Start pushing audio and video
      audioSender?.replaceTrack(audioTrack);
      videoSender?.replaceTrack(videoTrack);
    }
  };
};

Send and receive media data

The audio and video tracks added during connection setup (the RTP media channels) automatically transmit data to the server.

• Audio: transmitted directly over the audio track (RTP). No input_audio_buffer.append events are needed.

• Images: frames are sent over the video track (RTP). input_image_buffer.append events are not supported.

Demo source code