Integrate Queen SDK for Web

Initialize QueenEngine.

const sdkLicenseKey = ""; // The license key of Queen SDK.
const queenEngine = new QueenEngine();
//const queenEngine = new QueenEnginePro(); // Initialize Queen SDK Pro Edition.
queenEngine.create({
  SdkLicenseKey: sdkLicenseKey,
  OnInit: function(result: Boolean){
      console.info("queen sdk initialized.", result)
  },
  Domain: '',// Domain. You can use the root domain, such as aliyun.com.
  OnProgress: function(progress: Number){
      console.info("queen sdk loading:", progress);
  }
});

Initialize a specified canvas.

Set the canvasElement type parameter to webgl2 or leave it empty.

const canvasElement = document.getElementById("canvas");
const queenEngine = new QueenEngine();
queenEngine.create({
  SdkLicenseKey: sdkLicenseKey,
  OnInit: function(result: Boolean){
      console.info("queen sdk initialized.", result)  
  },
  Domain: '',// Domain. You can use the root domain.
  OnProgress: function(progress: Number){
      console.info("queen sdk loading:", progress);
  },
  RenderCanvas: canvasElement
});

Initialize specified inference models.

const queenEngine = new QueenEnginePro();
queenEngine.create({
  SdkLicenseKey: sdkLicenseKey,
  OnInit: function(result: Boolean){
      console.info("queen sdk initialized.", result)     
  },
  Domain: '',// Domain. You can use the root domain.
  OnProgress: function(progress: Number){
      console.info("queen sdk loading:", progress);
  },
  RenderCanvas: canvasElement// Optional.
  SegmentModel: kQueenModelShapeType.Vertical,// For portrait-oriented video. If your application does not require image matting, pass kQueenModelShapeType.None.
  PoseModel: kQueenModelShapeType.None,// Set to `None` to avoid loading body shaping model at initialization.
  InferenceBackend: kBackendType.WebGL,// The backend for model inference. In this example, WebGL is used.
});

Initialize a QueenEngineWorker instance.

import {QueenEngineWorker, kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBlend} from "aliyun-queen-engine"
const canvasElement = document.getElementById("canvas");
queenEngine = new QueenEngineWorker(kQueenVersion.Pro);
queenEngine.init(sdkLicenseKey, function(){
 // The initialization is complete.
}, function(progress){
 // progress: the loading progress.
}, canvasElement);

Configure basic face retouching.

// Enable basic retouching.
queenEngine.setQueenBeautyType(kQueenBeautyType.SkinBuffing, true);
// Set the level of skin smoothing.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinBuffing, 0.7);
// Set the level of image sharpening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Sharpen, 0.5);
// Enable skin whitening.
queenEngine.setQueenBeautyType(kQueenBeautyType.SkinWhiting, true);
// Set the level of skin whitening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Whitening, 0.6);
// Set the level of rosy cheeks.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinRed, 0.2);

Configure advanced face retouching.

// Enable advanced face retouching.
queenEngine.setQueenBeautyType(kQueenBeautyType.FaceBuffing, true);
// Set the level of eye bag removal.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Pouch, 0.9);
// Set the level of nasolabial fold removal.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.NasolabialFolds, 0.9);
// Set the level of teeth whitening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.WhiteTeeth, 0.9);
// Set the level of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Lipstick, 0.2);
// Set the level of blush.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Blush, 0.1);
// Set the color of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickColorParam, 0.1);
// Set the saturation of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickGlossParam, 0.1);
// Set the brightness of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickBrightnessParam, 0.1);
// Set the level of eye brightening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.BrightenEye, 0.9);
// Set the level of rosy cheeks.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinRed, 0.2);
// Set the level of wrinkle removal.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Wrinkles, 0.9);
// Set the level of skin brightening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.BrightenFace, 0.3);

Configure face shaping.

// Enable face shaping. The face shaping parameters are indicated by kQueenBeautyFaceShapeType.
queenEngine.setQueenBeautyType(kQueenBeautyType.FaceShape, true);
// Set the cheekbones. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.CutCheek, 0.6);
// Set cheekbone narrowing. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.CutFace, 0.7);
// Set face slimming. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.ThinFace, 0.8);
// Set the face length. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.LowerJaw, 0.8);
// Set chin lengthening. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.HigherJaw, 0.6); 

Configure makeup resources.

• Use the built-in makeup resources of the SDK.

  The SDK downloads retouching materials from Alibaba Cloud CDN when you use the built-in resources.

  // Enable makeup.
  queenEngine.setQueenBeautyType(kQueenBeautyType.Makeup, true);
  // Set the eyebrow effect.
  await queenEngine.setMakeupEyeBrow(Assets.kResMakeupEyeBrow.BiaoZhunMei, 0.6);
  // Set the eyelash effect.
  await queenEngine.setMackupEyeLash(Assets.kResMakeupEyeLash.ChenJing, 0.6);
  // Set the eye shadow effect.
  await queenEngine.setMakeupEyeShadow(Assets.kResMakeupEyeShadow.DaDiSe, 0.5);
  // Set the eyeliner effect.
  await queenEngine.setMakeupEyeLiner(Assets.kResMakeupEyeLiner.DaYan, 0.4);
  // Set the colored eye contacts effect.
  await queenEngine.setMakeupEyeBall(Assets.kResMakeupEyeBall.BiMuYu, 0.5);
  // Set the lipstick effect.
  await queenEngine.setMakeupMouth(Assets.kResMakeupMouth.AnYeZi, 0.3);
  // Set the blush effect.
  await queenEngine.setMakeupBlush(Assets.kResMakeupBlush.BlushWuGu, 0.2);
  // Set the highlight effect.
  await queenEngine.setMakeupHighlight(Assets.kResMakeupHighLight.Highlight, 0.1);
  // Remove the colored eye contacts effect.
  queenEngine.removeMakeupWithType(kQueenBeautyMakeupType.Eyeball);
  // For more information, see the QueenEngin.d.ts file.

• Use your local makeup resources.

  const makeupPackage = "./mouth.zip"
  const makeupName = "1.2.3.png";
  const band = kQueenBeautyBlend.LabMix;
  // Enable makeup. The makeup parameters are indicated by kQueenBeautyMakeupType.
  queenEngine.setQueenBeautyType(kQueenBeautyType.Makeup, true);
  // Set the transparency of the lipstick effect.
  queenEngine.setMakeupAlphaWithType(kQueenBeautyMakeupType.Mouth, true, 0.6);
  // Set the lipstick effect.
  queenEngine.setMakeupWithPackage(kQueenBeautyMakeupType.Mouth, makeupPackage, makeupName, band).then(() => {
  
  });

Configure filters.

• Use built-in filters.

  await queenEngine.setLutByType(Assets.kResLut.M1, 0.5);

• Use custom filters.

  const lutImageUrl = "./lut.png";
  queenEngine.setLutImageUrl(lutImageUrl).then(function () {
    queenEngine.setQueenBeautyType(kQueenBeautyType.LUT, true);
    queenEngine.setQueenBeautyParams(kQueenBeautyParams.LUT, 0.5);
  });

Configure stickers.

1. Use built-in stickers.

   queenEngine.addMaterialWithType(Assets.kResSticker.ILoveChina);
   // Specify a single sticker.
   queenEngine.addMaterialWithIndex(0);
   // Or, specify multiple stickers.
   queenEngine.addMaterialWithIndexs([0,1]);

2. Use custom stickers.

   const stickerZip = "./sticker.zip";
   queenEngine.addMaterialWithUrl(stickerZip).then(() => {
    });

Configure image matting.

const backgroundUrl = "./bg.png";
queenEngine.setSegmentBackgroundUrl(backgroundUrl).then(() => {
 });

Configure green-screen chroma keying.

const backgroundUrl = "./bg.png";
const isBlue = false;// Specify whether to use green-screen chroma keying.
queenEngine.setGreenScreenWithUrl(isBlue, backgroundUrl).then(() => {
 });

Configure background processing.

// Blur the background.
queenEngine.enableBokehBackground(true);
// Make the background transparent.
queenEngine.enableTransparentBackground(true);

Render the camera stream.

 queenEngine.openCameraAndRender().then((stream=>{
 const video = document.querySelector('video');
 video.srcObject = stream;
 video.play();
 })) 

Apply the rendered media to the canvas.

The canvas is the one specified during engine initialization.

 const sourceVideo = document.querySelector('video');
 queenEngine.renderMediaObjectToRenderCanvas(sourceVideo, sourceVideo.clientWidth, sourceVideo.clientHeight);

Before you call renderMediaObjectToRenderCanvas, make sure that the video element has finished loading. If the video has not finished loading, the SDK outputs the following error to the console, and the canvas displays no rendered output:

The Width and Height of the media object cannot be 0.

The SDK outputs this error only to the console by using console.error. No exception is thrown, so you must check the browser console to troubleshoot the issue.

To avoid this error, use one of the following approaches:

• Call the method in the loadedmetadata event callback:

  const sourceVideo = document.querySelector('video');
  sourceVideo.addEventListener('loadedmetadata', function() {
    if (sourceVideo.readyState >= 4 && sourceVideo.videoWidth > 0 && sourceVideo.videoHeight > 0) {
      queenEngine.renderMediaObjectToRenderCanvas(sourceVideo, sourceVideo.videoWidth, sourceVideo.videoHeight);
    }
  });

• Check the loading state before you call the method:

  if (sourceVideo.readyState >= 4 && sourceVideo.videoWidth > 0) {
    queenEngine.renderMediaObjectToRenderCanvas(sourceVideo, sourceVideo.videoWidth, sourceVideo.videoHeight);
  }

readyState >= 4 (HAVE_ENOUGH_DATA) indicates that the video can be played back safely.

• Record the rendered canvas.

  The resolution of the recorded video is determined by the width and height of the canvas. To control the resolution of the recorded video, set canvas.width and canvas.height to the target resolution before you call renderMediaObjectToRenderCanvas.

  // Set the canvas size to the target recording resolution.
  canvasElement.width = 640;
  canvasElement.height = 360;
  // Render the beautified video to the canvas.
  queenEngine.renderMediaObjectToRenderCanvas(sourceVideo, sourceVideo.clientWidth, sourceVideo.clientHeight);
  // Capture a media stream from the canvas. The resolution is 640 x 360.
  const recordStream = canvasElement.captureStream(30);
  // Record the stream by using MediaRecorder.
  const recorder = new MediaRecorder(recordStream, { mimeType: "video/webm" });

  Important

  Changing the canvas size after captureStream does not affect the stream that has already been created. You must set the canvas size before you call captureStream.

Render the video stream.

 navigator.mediaDevices.getUserMedia(constraints)
 .then(mediaStream => {
 let renderMediaStream = queenEngine.renderMediaStream(mediaStream);
 const video = document.querySelector('video');
 video.srcObject = renderMediaStream;
 video.play();
 });

Render an image.

fetch(Image URL)
.then(buffer => buffer.blob())
.then(createImageBitmap)
.then(img => {
 queenEngine.renderWithMediaObject(img, img.width, img.height, function(imageBufferData, imageWidth, imageHeight){
 const canvas = document.getElementById('playCanvas'); // The canvas. 
 const ctx = canvas.getContext("2d");
 const imageBuffer = new Uint8ClampedArray(imageBufferData);
 const imageData = new ImageData(imageBuffer, imageWidth, imageHeight);
 ctx.clearRect(0, 0, imageWidth, imageHeight);
 ctx.putImageData(imageData, 0, 0);
});
});

Render the texture.

const canvas = document.getElementById("sourceCanvas");
let outTexture = queenEngine.renderMediaObjectToTexture(canvas, canvas.width, canvas.height);
queenEngine.drawOutTexture(outTexture);// Draw the texture in the canvas that was specified during initialization.

Render the texture input.

let inputTexture = queenEngine.generateTextureId();
queenEngine.updateTexture(inputTexture, imageData);
let outTexture = queenEngine.renderTextureId(inputTexture, width, height);
queenEngine.drawOutTexture(outTexture)// Draw the texture in the canvas that was specified during initialization.

Render the pipeline stream.

const videoTrack = stream.getVideoTracks()[0];
const processor = new MediaStreamTrackProcessor({ track: videoTrack });
const readFrameStream = processor.readable;
const generator = new MediaStreamTrackGenerator({ kind: 'video' });
let writeFrameStream = generator.writable;
readFrameStream.pipeThrough(queenEngine.getTransformStream()).pipeTo(writeFrameStream);