带标量过滤条件的向量检索
更新时间：2024-12-24
什么是Filter过滤条件
VectorDB支持通过表达式形式指定一个过滤条件作为检索参数，来约束向量检索的候选数据范围，这就是我们常说的带标量过滤的向量检索。本文介绍一下此类检索场景的应进行一些说明介绍。其中Filter过滤条件表达式的语法可以参见Filter过滤条件表达式语法说明。
带Filter过滤条件的向量检索示例
我们通过Python SDK来展示一系列带Filter过滤条件表达式的向量检索的使用方法。
- 创建一个莫愁客户端对象。
import time
import json
import random
import pymochow
import logging
from pymochow.configuration import Configuration
from pymochow.auth.bce_credentials import BceCredentials
from pymochow.exception import ClientError, ServerError
from pymochow.model.schema import Schema, Field, SecondaryIndex, VectorIndex, HNSWParams
from pymochow.model.enum import FieldType, IndexType, MetricType, ServerErrCode
from pymochow.model.enum import TableState, IndexState
from pymochow.model.table import Partition, Row, AnnSearch, HNSWSearchParams
logging.basicConfig(filename='documentinsight.log', level=logging.DEBUG,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
if __name__ == "__main__":
account = '$您的账户名称'
api_key = '$您的账户API密钥'
endpoint = '$您的实例访问端点' # 例如：'http://127.0.0.1:5287'
db_name = 'DocumentInsight'
table_name = 'FilterSearchTable'
# 根据配置创建一个MochowClient对象
config = Configuration(credentials=BceCredentials(account, api_key), endpoint=endpoint)
mochow_client = pymochow.MochowClient(config)- 建库建表，建立一个有着各种不同类型字段的表。
# 建库
db = mochow_client.create_database(db_name);
# 若库已存在，则直接使用下述代码获取一个库对象
# db = mochow_client.database(db_name)
# 建表，该表有2个分区
fields = []
fields.append(Field("DocId", FieldType.UINT64, primary_key=True, partition_key=True, not_null=True))
fields.append(Field("Title", FieldType.STRING, not_null=True))
fields.append(Field("Author", FieldType.STRING, not_null=True))
fields.append(Field("Extension", FieldType.STRING, not_null=True))
fields.append(Field("SizeInBytes", FieldType.UINT64, not_null=True))
fields.append(Field("LastEditTime", FieldType.DATETIME, not_null=True))
fields.append(Field("Published", FieldType.BOOL, not_null=True))
fields.append(Field("Vector", FieldType.FLOAT_VECTOR, dimension=2, not_null=True))
indexes = []
indexes.append(VectorIndex(index_name="Vector_Idx", index_type=IndexType.HNSW,
field="Vector", metric_type=MetricType.L2,
params=HNSWParams(m=32, efconstruction=200)))
table = db.create_table(
table_name=table_name,
replication=1,
partition=Partition(partition_num=2), # 2个分区
schema=Schema(fields=fields, indexes=indexes)
)
while True:
time.sleep(1)
table = db.describe_table(table_name)
if table.state == TableState.NORMAL:
break- 插入数条记录，仔细设计这些记录的各个字段的取值，便于我们后续展示各种带Filter条件表达式的查询。
# 插入数条数据，为了方便展示带过滤检索的用法，我们让这些数据具备非常接近的向量
rows = [
Row(DocId=1, Title="LLM技术详解", Author='Alice', Extension='pdf', SizeInBytes=100000,
LastEditTime="2024-01-28T12:55:28Z", Published=True, Vector=[0.111111, 0.222222]),
Row(DocId=2, Title="深入浅出大语言模型技术", Author='Bob', Extension='pdf', SizeInBytes=110000,
LastEditTime="2024-02-29T18:00:00Z", Published=True, Vector=[0.1111112, 0.222222]),
Row(DocId=3, Title="基于大模型的RAG应用开发实践", Author='Bob', Extension='docx', SizeInBytes=120000,
LastEditTime="2023-11-09T11:00:00Z", Published=True, Vector=[0.111113, 0.222222]),
Row(DocId=4, Title="揭秘AI原生应用开发——原理与实战", Author='Curry', Extension='docx', SizeInBytes=120000,
LastEditTime="2023-12-12T09:40:30Z", Published=False, Vector=[0.111114, 0.222222]),
Row(DocId=5, Title="向量数据库在RAG开发中的应用与实践", Author='Curry', Extension='pptx', SizeInBytes=90000,
LastEditTime="2024-02-29T18:00:00Z", Published=False, Vector=[0.111115, 0.222222])
]
table.insert(rows=rows)- 构建向量索引。
# 构建向量索引
index_name = "Vector_Idx"
table.rebuild_index(index_name)
while True:
time.sleep(1)
index = table.describe_index(index_name)
if index.state == IndexState.NORMAL:
break- 执行各类带Filter过滤条件表达式的向量检索，在所有检索示例中，我们都检索非常接近上述数据向量值的同一个向量值。
- Filter为单一表达式
vectorFloats = [0.1, 0.2]
# 仅在“Author为Bob”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="Author == 'Bob'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"Author == 'Bob'\": {}\n".format(result))
# 仅在“Published为True”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="Published = 'true'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"Published = 'true'\": {}\n".format(result))
# 仅在“DocID大于等于3”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="DocId >= 3",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"DocId >= 3\": {}\n".format(result))
# 仅在“LastEditTime小于UTC时间2024年1月1日0点0分0秒（即为2024年之前）”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="LastEditTime < '2024-01-01T00:00:00Z'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"LastEditTime < '2024-01-01T00:00:00Z'\": {}\n".format(result))
# 仅在“Extension不是pptx”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="Extension != 'pptx'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"Extension != 'pptx'\": {}\n".format(result))
# 仅在“Extension是pptx这个条件为假”的数据记录中进行检索，该条件等价于上一个条件，即“Extension不是pptx”
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="NOT Extension == 'pptx'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"NOT Extension == 'pptx'\": {}\n".format(result))- Filter为简单组合表达式
vectorFloats = [0.1, 0.2]
# 仅在“Author为Curry 且 LastEditTime大于UTC时间2024年1月1日0点0分0秒（即为2024年）”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="Author == 'Curry' AND LastEditTime > '2024-01-01T00:00:00Z'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"Author == 'Curry' AND LastEditTime > '2024-01-01T00:00:00Z'\": {}\n".format(result))
# 仅在“Extension为pdf 或 SizeInBytes等于120000”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="Extension == 'pdf' OR SizeInBytes == 120000",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"Extension == 'pdf' OR SizeInBytes == 120000\": {}\n".format(result))
# 仅在“Extension为docx 或 Extension为pptx”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="Extension == 'docx' OR Extension = 'pptx'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"Extension == 'docx' OR Extension = 'pptx'\": {}\n".format(result))- Filter为复杂组合表达式
vectorFloats = [0.1, 0.2]
# 仅在“Extension为pdf或docx，并且Author为Curry”的数据记录中进行检索
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="(Extension == 'pdf' OR Extension = 'docx') AND Author == 'Curry'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"(Extension == 'pdf' OR Extension = 'docx') AND Author == 'Curry'\": {}\n".format(result))
# Filter条件过于复杂，此处不在用中文进行描述
anns = AnnSearch(vector_field="Vector", vector_floats=vectorFloats,
filter="((Extension == 'pdf' OR LastEditTime < '2024-01-01T00:00:00Z') AND Author == 'Curry') OR Title == 'LLM技术详解'",
params=HNSWSearchParams(ef=200, limit=10))
result = table.search(anns=anns, retrieve_vector=True)
print("Search result with filter \"((Extension == 'pdf' OR LastEditTime < '2024-01-01T00:00:00Z') AND Author == 'Curry') OR Title == 'LLM技术详解'\": {}\n".format(result))- 上述所有检索请求的全部输出结果，如下所示：
Search result with filter "Author == 'Bob'": {metadata:{content__length:u'644',content__type:u'application/json',request_id:u'b2b91133-13a4-4ce6-8533-0a73e9f8f782'},rows:[{'row': {'DocId': 2, 'Vector': [0.11111120134592056, 0.2222220003604889], 'Title': '深入浅出大语言模型技术', 'Author': 'Bob', 'Extension': 'pdf', 'SizeInBytes': 110000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': True}, 'distance': 0.0006172759458422661, 'score': 1.0}, {'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "Published = 'true'": {metadata:{content__length:u'929',content__type:u'application/json',request_id:u'accff13e-c19b-4356-9181-da6fac2bdebb'},rows:[{'row': {'DocId': 1, 'Vector': [0.11111100018024445, 0.2222220003604889], 'Title': 'LLM技术详解', 'Author': 'Alice', 'Extension': 'pdf', 'SizeInBytes': 100000, 'LastEditTime': '2024-01-28T12:55:28Z', 'Published': True}, 'distance': 0.0006172714056447148, 'score': 1.0}, {'row': {'DocId': 2, 'Vector': [0.11111120134592056, 0.2222220003604889], 'Title': '深入浅出大语言模型技术', 'Author': 'Bob', 'Extension': 'pdf', 'SizeInBytes': 110000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': True}, 'distance': 0.0006172759458422661, 'score': 1.0}, {'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "DocId >= 3": {metadata:{content__length:u'977',content__type:u'application/json',request_id:u'c3a6fb0b-3ea2-4bb3-8897-654541eb91bf'},rows:[{'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}, {'row': {'DocId': 5, 'Vector': [0.111115001142025, 0.2222220003604889], 'Title': '向量数据库在RAG开发中的应用与实践', 'Author': 'Curry', 'Extension': 'pptx', 'SizeInBytes': 90000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': False}, 'distance': 0.0006173603469505906, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "LastEditTime < '2024-01-01T00:00:00Z'": {metadata:{content__length:u'660',content__type:u'application/json',request_id:u'bc716bae-6b70-4faf-b1ba-d434ff2c85dd'},rows:[{'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "Extension != 'pptx'": {metadata:{content__length:u'1246',content__type:u'application/json',request_id:u'282f60ff-6f4c-4b5e-b533-f3aadb518c02'},rows:[{'row': {'DocId': 1, 'Vector': [0.11111100018024445, 0.2222220003604889], 'Title': 'LLM技术详解', 'Author': 'Alice', 'Extension': 'pdf', 'SizeInBytes': 100000, 'LastEditTime': '2024-01-28T12:55:28Z', 'Published': True}, 'distance': 0.0006172714056447148, 'score': 1.0}, {'row': {'DocId': 2, 'Vector': [0.11111120134592056, 0.2222220003604889], 'Title': '深入浅出大语言模型技术', 'Author': 'Bob', 'Extension': 'pdf', 'SizeInBytes': 110000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': True}, 'distance': 0.0006172759458422661, 'score': 1.0}, {'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "NOT Extension == 'pptx'": {metadata:{content__length:u'1246',content__type:u'application/json',request_id:u'8ac05714-af05-4d53-9c25-740e3acf73b8'},rows:[{'row': {'DocId': 1, 'Vector': [0.11111100018024445, 0.2222220003604889], 'Title': 'LLM技术详解', 'Author': 'Alice', 'Extension': 'pdf', 'SizeInBytes': 100000, 'LastEditTime': '2024-01-28T12:55:28Z', 'Published': True}, 'distance': 0.0006172714056447148, 'score': 1.0}, {'row': {'DocId': 2, 'Vector': [0.11111120134592056, 0.2222220003604889], 'Title': '深入浅出大语言模型技术', 'Author': 'Bob', 'Extension': 'pdf', 'SizeInBytes': 110000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': True}, 'distance': 0.0006172759458422661, 'score': 1.0}, {'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "Author == 'Curry' AND LastEditTime > '2024-01-01T00:00:00Z'": {metadata:{content__length:u'352',content__type:u'application/json',request_id:u'c05d60b4-12a3-449a-962a-db545794c67f'},rows:[{'row': {'DocId': 5, 'Vector': [0.111115001142025, 0.2222220003604889], 'Title': '向量数据库在RAG开发中的应用与实践', 'Author': 'Curry', 'Extension': 'pptx', 'SizeInBytes': 90000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': False}, 'distance': 0.0006173603469505906, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "Extension == 'pdf' OR SizeInBytes == 120000": {metadata:{content__length:u'1246',content__type:u'application/json',request_id:u'0a360057-da65-4a05-957d-c486bc2f97f6'},rows:[{'row': {'DocId': 1, 'Vector': [0.11111100018024445, 0.2222220003604889], 'Title': 'LLM技术详解', 'Author': 'Alice', 'Extension': 'pdf', 'SizeInBytes': 100000, 'LastEditTime': '2024-01-28T12:55:28Z', 'Published': True}, 'distance': 0.0006172714056447148, 'score': 1.0}, {'row': {'DocId': 2, 'Vector': [0.11111120134592056, 0.2222220003604889], 'Title': '深入浅出大语言模型技术', 'Author': 'Bob', 'Extension': 'pdf', 'SizeInBytes': 110000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': True}, 'distance': 0.0006172759458422661, 'score': 1.0}, {'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "Extension == 'docx' OR Extension = 'pptx'": {metadata:{content__length:u'977',content__type:u'application/json',request_id:u'e3111372-ffb7-4f74-af59-ce961fc3ff90'},rows:[{'row': {'DocId': 3, 'Vector': [0.11111299693584442, 0.2222220003604889], 'Title': '基于大模型的RAG应用开发实践', 'Author': 'Bob', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-11-09T11:00:00Z', 'Published': True}, 'distance': 0.0006173158180899918, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}, {'row': {'DocId': 5, 'Vector': [0.111115001142025, 0.2222220003604889], 'Title': '向量数据库在RAG开发中的应用与实践', 'Author': 'Curry', 'Extension': 'pptx', 'SizeInBytes': 90000, 'LastEditTime': '2024-02-29T18:00:00Z', 'Published': False}, 'distance': 0.0006173603469505906, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "(Extension == 'pdf' OR Extension = 'docx') AND Author == 'Curry'": {metadata:{content__length:u'352',content__type:u'application/json',request_id:u'2604c2af-83a6-43f1-9c4a-9e843d69a544'},rows:[{'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}],code:0,msg:u'Success'}
Search result with filter "((Extension == 'pdf' OR LastEditTime < '2024-01-01T00:00:00Z') AND Author == 'Curry') OR Title == 'LLM技术详解'": {metadata:{content__length:u'637',content__type:u'application/json',request_id:u'5f0ae352-5c44-4193-a74d-ef2494a76038'},rows:[{'row': {'DocId': 1, 'Vector': [0.11111100018024445, 0.2222220003604889], 'Title': 'LLM技术详解', 'Author': 'Alice', 'Extension': 'pdf', 'SizeInBytes': 100000, 'LastEditTime': '2024-01-28T12:55:28Z', 'Published': True}, 'distance': 0.0006172714056447148, 'score': 1.0}, {'row': {'DocId': 4, 'Vector': [0.111114002764225, 0.2222220003604889], 'Title': '揭秘AI原生应用开发——原理与实战', 'Author': 'Curry', 'Extension': 'docx', 'SizeInBytes': 120000, 'LastEditTime': '2023-12-12T09:40:30Z', 'Published': False}, 'distance': 0.0006173381698317826, 'score': 1.0}],code:0,msg:u'Success'}