Vectorize multi-modal data by using open source multi-modal embedding models of ModelScope

Step 1: Set up the embedding pipeline and DashVector client

from modelscope.utils.constant import Tasks
from modelscope.pipelines import pipeline
from modelscope.preprocessors.image import load_image
from typing import List
from dashvector import Client

# Load the CLIP model pipeline
clip_pipeline = pipeline(task=Tasks.multi_modal_embedding, model='<model-id>')


def generate_text_embeddings(texts: List[str]):
    inputs = {'text': texts}
    result = clip_pipeline.forward(input=inputs)
    return result['text_embedding'].numpy()


def generate_img_embeddings(img: str):
    input_img = load_image(img)
    inputs = {'img': input_img}
    result = clip_pipeline.forward(input=inputs)
    return result['img_embedding'].numpy()[0]


# Initialize the DashVector client
client = Client(
    api_key='<your-dashvector-api-key>',
    endpoint='<your-dashvector-cluster-endpoint>'
)

Step 2: Create a collection

Text and image vectors from the same CLIP model share the same dimension, so you can store both in a single collection and search across modalities.

# Create a collection with the dimension matching your chosen CLIP model
rsp = client.create('CLIP-embedding', dimension=<model-dim>)
assert rsp
collection = client.get('CLIP-embedding')
assert collection

Step 3: Insert text and image vectors

# Generate embeddings and insert them into the collection
collection.insert(
    [
        ('ID1', generate_text_embeddings(['DashVector developed by Alibaba Cloud is one of the most high-performance and cost-effective vector database services.'])[0]),
        ('ID2', generate_img_embeddings('https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg'))
    ]
)

Step 4: Run a cross-modal vector search

Use a text query to search across both text and image vectors in the collection.

# Query with text to find semantically similar results across all modalities
docs = collection.query(
    generate_text_embeddings(['The best vector database'])[0]
)
print(docs)