通过Python（OpenSearch Python）访问向量引擎

插入写

使用index写入，若目标数据已存在，则不允许写入。以写入索引vector_test为例：

doc = {
  "field1": 1,
  "vector1": [1.2, 1.3, 1.4]
}
response = client.index(index='vector_test', body=doc, id=1)

更新写

使用create写入，若目标数据已存在，则执行更新操作；若目标数据不存在，写入数据。以写入索引vector_test为例：

doc = {
  "field1": 2,
  "vector1": [2.2, 2.3, 2.4]
}
response = client.create(index='vector_test', body=doc, id=2)

通用写入

以写入索引vector_test为例：

operations = """
{ "index" : { "_index" : "vector_test", "_id" : "2" } }
{ "field1" : 1, "vector1": [2.2, 2.3, 2.4]}
{ "create" : { "_index" : "vector_test", "_id" : "3" } }
{ "field1" : 2, "vector1": [1.2, 1.3, 4.4]}
{ "delete" : { "_index" : "vector_test", "_id" : "2" } }
{ "update" : {"_id" : "1", "_index" : "vector_test"} }
{ "doc" : {"field1" : 3, "vector1": [2.2, 3.3, 4.4]} }
"""
response = client.bulk(operations)

# 批量数据写入封装
def write_docs(self, index_name: str) -> None:
    operations = []
    for i in range(0, 1000):
        id = self.random.randint(-2 ** 63, 2 ** 63 - 1)
        operations.append(json.dumps({"index": {"_index": index_name, "_id": id}}))
        operations.append("\n")
        vector1 = []
        for j in range(0, 5):
            vector1.append(self.random.random())
        operations.append(json.dumps({"field1": self.random.random(), "vector1": vector1}))
        operations.append("\n")
    response = self.client.bulk("".join(operations))

稀疏向量写入

写入方式与上述方式相同，但需要修改vector1的格式。共支持两种格式：JSON STRING和JSON Object。前者性能更优，后者格式更友好。

doc = {
  "field1": 2,
  "vector1": "{\"indices\": [10, 14, 16], \"values\": [0.5, 0.5, 0.2]}"
}
response = client.index(index='vector_sparse_test', body=doc, id=1)

doc = {
  "field1": 1,
  "vector1": {"indices": [10, 12, 16], "values": [0.5, 0.5, 0.2]}
}
response = client.index(index='vector_sparse_test', body=doc, id=1)

通用查询方式

以下示例查询索引vector_test的向量列vector1字段中与向量[2.3, 3.3, 4.4]相关的前10条数据，并要求最小得分为0.1。

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "k": 10
      }
    }
  },
  "ext": {"lvector": {"min_score": "0.1"}}
}
response = client.search(index='vector_test', body=query)

参数说明

返回结果如下：

{
  "took" : 30,
  "timed_out" : false,
  "terminated_early" : false,
  "num_reduce_phases" : 0,
  "_shards" : {
    "total" : 2,
    "successful" : 2,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 3,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "vector_test",
        "_id" : "2",
        "_score" : 1.0
      },
      {
        "_index" : "vector_test",
        "_id" : "1",
        "_score" : 0.25
      },
      {
        "_index" : "vector_test",
        "_id" : "3",
        "_score" : 0.14285715
      }
    ]
  }
}

返回指定字段

如果需要在查询时返回指定字段，可以指定 "_source": ["field1", "field2"] 或使用"_source": true 返回非向量的全部字段。以查询索引vector_test为例，使用方法如下：

query = {
  "size": 10,
  "_source": ["field1"],
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "k": 10
      }
    }
  },
  "ext": {"lvector": {"min_score": "0.1"}}
}
response = client.search(index='vector_test', body=query)

返回结果如下：

{
  "took" : 35,
  "timed_out" : false,
  "terminated_early" : false,
  "num_reduce_phases" : 0,
  "_shards" : {
    "total" : 2,
    "successful" : 2,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 3,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "vector_test",
        "_id" : "2",
        "_score" : 1.0,
        "_source" : {
          "field1" : 1
        }
      },
      {
        "_index" : "vector_test",
        "_id" : "1",
        "_score" : 0.25,
        "_source" : {
          "field1" : 1
        }
      },
      {
        "_index" : "vector_test",
        "_id" : "3",
        "_score" : 0.14285715,
        "_source" : {
          "field1" : 2
        }
      }
    ]
  }
}

hsnw算法查询

以查询hsnw索引vector_test为例：

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "k": 10
      }
    }
  },
  "ext": {"lvector": {"ef_search": "100"}}
}
response = client.search(index='vector_test', body=query)

ivfpq算法查询

以查询ivfpq索引vector_ivfpq_test为例：

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "k": 10
      }
    }
  },
  "ext": {"lvector": {"nprobe": "60", "reorder_factor": "5"}}
}
response = client.search(index='vector_ivfpq_test', body=query)

稀疏向量查询

查询方式与上述方式相同，但需要修改vector1的格式。共支持两种格式：JSON STRING和JSON Object。前者性能更优，后者格式更友好。

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": "{\"indices\": [10, 45, 16], \"values\": [0.5, 0.5, 0.2]}",
        "k": 10
      }
    }
  }
}
response = client.search(index='vector_sparse_test', body=query)

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": {"indices": [10, 45, 16], "values": [0.5, 0.5, 0.2]},
        "k": 10
      }
    }
  }
}
response = client.search(index='vector_sparse_test', body=query)

Pre-Filter近似查询

通过在knn查询结构内部添加过滤器filter，并指定filter_type参数的值为pre_filter，可实现先过滤结构化数据，再查询向量数据。

以查询索引vector_test为例：

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "filter": {
          "range": {
            "field1": {
              "gte": 0
            }
          }
        },
        "k": 10
      }
    }
  },
  "ext": {"lvector": {"filter_type": "pre_filter"}}
}
response = client.search(index='vector_test', body=query)

Post-Filter近似查询

通过在knn查询结构内部添加过滤器filter，并指定filter_type参数的值为post_filter，可实现先查询向量数据，再过滤结构化数据。

以查询索引vector_test为例：

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "filter": {
          "range": {
            "field1": {
              "gte": 0
            }
          }
        },
        "k": 10
      }
    }
  },
  "ext": {"lvector": {"filter_type": "post_filter"}}
}
response = client.search(index='vector_test', body=query)

在使用Post_Filter近似查询时需要适当放大k的值，如果使用ivfpq算法，还需要调整reorder_factor的值。以查询索引vector_test为例，具体使用如下：

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "filter": {
          "range": {
            "field1": {
              "gte": 0
            }
          }
        },
        "k": 1000
      }
    }
  },
  "ext": {"lvector": {"filter_type": "post_filter","nprobe": "60", "reorder_factor": "1"}}
}
response = client.search(index='vector_test', body=query)

您也可以通过post_filter添加过滤条件，实现Post-Filter近似查询。以查询索引vector_test为例，具体使用如下：

query = {
  "size": 10,
  "query": {
    "knn": {
      "vector1": {
        "vector": [2.3, 3.3, 4.4],
        "k": 10
      }
    }
  },
  "post_filter": {
    "range": {
      "field1": {
        "gte": 0
      }
    }
  }
}
response = client.search(index='vector_test', body=query)