Grouped document search

Search by vector

This example creates a collection, inserts documents with document_id and chunk_id fields, and runs a grouped search using a query vector. The results are grouped by document_id, returning up to 2 groups with up to 3 documents each.

package com.aliyun.dashvector;

import com.aliyun.dashvector.models.Doc;
import com.aliyun.dashvector.models.Group;
import com.aliyun.dashvector.models.Vector;
import com.aliyun.dashvector.models.requests.CreateCollectionRequest;
import com.aliyun.dashvector.models.requests.InsertDocRequest;
import com.aliyun.dashvector.models.requests.QueryDocGroupByRequest;
import com.aliyun.dashvector.models.responses.Response;
import com.aliyun.dashvector.proto.CollectionInfo;
import com.aliyun.dashvector.proto.FieldType;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;

public class Main {
    public static void main(String[] args) {
        DashVectorClientConfig config =
                DashVectorClientConfig.builder()
                        .apiKey("YOUR_API_KEY")
                        .endpoint("YOUR_CLUSTER_ENDPOINT")
                        .timeout(30f)
                        .build();
        DashVectorClient client = new DashVectorClient(config);

        CreateCollectionRequest request = CreateCollectionRequest.builder()
                .name("group_demo")
                .dimension(4)
                .metric(CollectionInfo.Metric.dotproduct)
                .dataType(CollectionInfo.DataType.FLOAT)
                .filedSchema("document_id", FieldType.STRING)
                .filedSchema("chunk_id", FieldType.INT)
                .build();

        Response<Void> createResponse = client.create(request);
        assert createResponse.isSuccess();

        DashVectorCollection collection = client.get("group_demo");
        assert collection.isSuccess();

        List<Doc> docs = new ArrayList<>();
        for (int i = 0; i < 6; i++) {
            int finalI = i;
            docs.add(
                    Doc.builder()
                            .id(Integer.toString(i+3))
                            .vector(Vector.builder().value(Collections.nCopies(4, (float) i+3)).build())
                            .fields(
                                    new HashMap<String, Object>() {
                                        {
                                            put("document_id", "paper" + finalI % 3);
                                            put("chunk_id", finalI);
                                        }
                                    }
                            )
                            .build());
        }

        InsertDocRequest insertRequest = InsertDocRequest.builder().docs(docs).build();
        Response<Void> insertResponse = collection.insert(insertRequest);
        assert insertResponse.isSuccess();

        // Build a query vector.
        Vector queryVector = Vector.builder().value(Arrays.asList(0.1f, 0.2f, 0.3f, 0.4f)).build();

        // Build the request: group by document_id, return 2 groups with up to 3 docs each.
        QueryDocGroupByRequest queryDocGroupByRequest =
                QueryDocGroupByRequest.builder()
                        .includeVector(true)
                        .vector(queryVector)
                        .groupByField("document_id")
                        .groupCount(2)
                        .groupTopk(3)
                        .build();

        // Run the grouped search.
        Response<List<Group>> response = collection.queryGroupBy(queryDocGroupByRequest);
        assert response.isSuccess();

        System.out.println(response);
    }
}

The response groups results by document_id. Each group contains the most similar chunks from that document:

{
    "code": 0,
    "message": "Success",
    "requestId": "47404048-6f40-47ad-9d62-2675704afb26",
    "output": [
        {
            "groupId": "paper2",
            "docs": [
                {
                    "id": "8",
                    "vector": { "value": [8.0, 8.0, 8.0, 8.0] },
                    "fields": { "document_id": "paper2", "chunk_id": 5 },
                    "score": 8.0,
                    "sparseVector": {}
                },
                {
                    "id": "5",
                    "vector": { "value": [5.0, 5.0, 5.0, 5.0] },
                    "fields": { "document_id": "paper2", "chunk_id": 2 },
                    "score": 5.0,
                    "sparseVector": {}
                }
            ]
        },
        {
            "groupId": "paper1",
            "docs": [
                {
                    "id": "7",
                    "vector": { "value": [7.0, 7.0, 7.0, 7.0] },
                    "fields": { "document_id": "paper1", "chunk_id": 4 },
                    "score": 7.0,
                    "sparseVector": {}
                },
                {
                    "id": "4",
                    "vector": { "value": [4.0, 4.0, 4.0, 4.0] },
                    "fields": { "document_id": "paper1", "chunk_id": 1 },
                    "score": 4.0,
                    "sparseVector": {}
                }
            ]
        }
    ]
}

Search by primary key

To search using the vector stored with a specific document, specify the document's primary key using id() instead of vector(). The search finds similar documents based on that stored vector and groups the results.

// Build the request: use the vector of document "1", group by age field.
QueryDocGroupByRequest request = QueryDocGroupByRequest.builder()
        .groupByField("age")
        .id("1")                                             // Use the vector stored for this ID.
        .groupCount(10)                                      // Return up to 10 groups.
        .groupTopk(10)                                       // Return up to 10 documents per group.
        .outputFields(Arrays.asList("name", "age"))          // Return only name and age fields.
        .includeVector(true)
        .build();

Response<List<Group>> response2 = collection.queryGroupBy(request);
assert response2.isSuccess();

Search with a filter

Add a filter() to restrict the search to documents matching a condition. The filter uses SQL WHERE clause syntax.

Vector vector = Vector.builder().value(Arrays.asList(0.1f, 0.2f, 0.3f, 0.4f)).build();

// Build the request: search by vector with a filter on the age field.
QueryDocGroupByRequest request = QueryDocGroupByRequest.builder()
        .groupByField("age")
        .vector(vector)
        .filter("age > 18")                                  // Only include documents where age > 18.
        .outputFields(Arrays.asList("name", "age"))
        .includeVector(true)
        .build();

Response<List<Group>> response = collection.queryGroupBy(request);
assert response.isSuccess();

For filter syntax details, see Conditional filtering.

Search with dense and sparse vectors

Combine a dense vector with a sparse vector to implement keyword-aware semantic search. You can use a sparse vector to represent the keyword weight.

For more information about keyword-aware semantic search, see Keywords-aware retrieval.

Vector vector = Vector.builder().value(Arrays.asList(0.1f, 0.2f, 0.3f, 0.4f)).build();

// Build the request: combine dense and sparse vectors for hybrid grouped search.
QueryDocGroupByRequest request = QueryDocGroupByRequest.builder()
        .groupByField("age")
        .vector(vector)
        .sparseVector(
                new Map<Integer, Float>() {
                    {
                        put(1, 0.4f);
                        put(10000, 0.6f);
                        put(222222, 0.8f);
                    }
                })
        .build();

Response<List<Group>> response = collection.queryGroupBy(request);
assert response.isSuccess();