Support TypeParams in STRUCT [2.6] (#3055)

pr: #3054
issue: milvus-io/milvus#42148

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>

Author: SpadeA-Tang

pymilvus/client/abstract.py

@@ -145,6 +145,7 @@ def __init__(self, raw: Any):
         self.name = None
         self.fields = []
         self.description = None
+        self.params = {}
 
         self.__pack(self._raw)
 
@@ -154,14 +155,26 @@ def __pack(self, raw: Any):
         self.description = raw.description
         self.fields = [FieldSchema(f) for f in raw.fields]
 
+        self.params = {}
+        for kv in raw.type_params:
+            key = kv.key if kv.key != "mmap.enabled" else "mmap_enabled"
+            try:
+                value = orjson.loads(kv.value)
+            except (orjson.JSONDecodeError, ValueError):
+                value = kv.value
+            self.params[key] = value
+
     def dict(self):
-        return {
+        result = {
             "field_id": self.field_id,
             "name": self.name,
             "description": self.description,
             "type": DataType._ARRAY_OF_STRUCT,
             "fields": [f.dict() for f in self.fields],
         }
+        if self.params:
+            result["params"] = self.params
+        return result
 
 
 class FunctionSchema:

pymilvus/client/prepare.py

@@ -182,6 +182,19 @@ def get_schema_from_collection_schema(
                 fields=[],
                 description=struct.description,
             )
+
+            if struct.params:
+                for k, v in struct.params.items():
+                    kv_pair = common_types.KeyValuePair(
+                        key=str(k) if k != "mmap_enabled" else "mmap.enabled",
+                        value=(
+                            orjson.dumps(v).decode(Config.EncodeProtocol)
+                            if not isinstance(v, str)
+                            else str(v)
+                        ),
+                    )
+                    struct_schema.type_params.append(kv_pair)
+
             for f in struct.fields:
                 # Convert struct field types to backend representation
                 # As struct itself only support array type, so all it's fields are array type

pymilvus/grpc_gen/common_pb2.py

@@ -138,6 +138,9 @@ class MsgType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     DescribeAlias: _ClassVar[MsgType]
     ListAliases: _ClassVar[MsgType]
     AlterCollectionField: _ClassVar[MsgType]
+    AddCollectionFunction: _ClassVar[MsgType]
+    AlterCollectionFunction: _ClassVar[MsgType]
+    DropCollectionFunction: _ClassVar[MsgType]
     CreatePartition: _ClassVar[MsgType]
     DropPartition: _ClassVar[MsgType]
     HasPartition: _ClassVar[MsgType]
@@ -351,6 +354,9 @@ class ObjectPrivilege(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     PrivilegeAddFileResource: _ClassVar[ObjectPrivilege]
     PrivilegeRemoveFileResource: _ClassVar[ObjectPrivilege]
     PrivilegeListFileResources: _ClassVar[ObjectPrivilege]
+    PrivilegeAddCollectionFunction: _ClassVar[ObjectPrivilege]
+    PrivilegeAlterCollectionFunction: _ClassVar[ObjectPrivilege]
+    PrivilegeDropCollectionFunction: _ClassVar[ObjectPrivilege]
 
 class StateCode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
@@ -372,10 +378,6 @@ class LoadPriority(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     HIGH: _ClassVar[LoadPriority]
     LOW: _ClassVar[LoadPriority]
 
-class FileResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
-    __slots__ = ()
-    ANALYZER_DICTIONARY: _ClassVar[FileResourceType]
-
 class WALName(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
     Unknown: _ClassVar[WALName]
@@ -384,6 +386,11 @@ class WALName(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     Kafka: _ClassVar[WALName]
     WoodPecker: _ClassVar[WALName]
     Test: _ClassVar[WALName]
+
+class HighlightType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    Lexical: _ClassVar[HighlightType]
+    Semantic: _ClassVar[HighlightType]
 Success: ErrorCode
 UnexpectedError: ErrorCode
 ConnectFailed: ErrorCode
@@ -498,6 +505,9 @@ RenameCollection: MsgType
 DescribeAlias: MsgType
 ListAliases: MsgType
 AlterCollectionField: MsgType
+AddCollectionFunction: MsgType
+AlterCollectionFunction: MsgType
+DropCollectionFunction: MsgType
 CreatePartition: MsgType
 DropPartition: MsgType
 HasPartition: MsgType
@@ -693,6 +703,9 @@ PrivilegeAddCollectionField: ObjectPrivilege
 PrivilegeAddFileResource: ObjectPrivilege
 PrivilegeRemoveFileResource: ObjectPrivilege
 PrivilegeListFileResources: ObjectPrivilege
+PrivilegeAddCollectionFunction: ObjectPrivilege
+PrivilegeAlterCollectionFunction: ObjectPrivilege
+PrivilegeDropCollectionFunction: ObjectPrivilege
 Initializing: StateCode
 Healthy: StateCode
 Abnormal: StateCode
@@ -704,13 +717,14 @@ LoadStateLoading: LoadState
 LoadStateLoaded: LoadState
 HIGH: LoadPriority
 LOW: LoadPriority
-ANALYZER_DICTIONARY: FileResourceType
 Unknown: WALName
 RocksMQ: WALName
 Pulsar: WALName
 Kafka: WALName
 WoodPecker: WALName
 Test: WALName
+Lexical: HighlightType
+Semantic: HighlightType
 PRIVILEGE_EXT_OBJ_FIELD_NUMBER: _ClassVar[int]
 privilege_ext_obj: _descriptor.FieldDescriptor
 
@@ -978,3 +992,25 @@ class ReplicateCheckpoint(_message.Message):
     message_id: MessageID
     time_tick: int
     def __init__(self, cluster_id: _Optional[str] = ..., pchannel: _Optional[str] = ..., message_id: _Optional[_Union[MessageID, _Mapping]] = ..., time_tick: _Optional[int] = ...) -> None: ...
+
+class HighlightData(_message.Message):
+    __slots__ = ("fragments",)
+    FRAGMENTS_FIELD_NUMBER: _ClassVar[int]
+    fragments: _containers.RepeatedScalarFieldContainer[str]
+    def __init__(self, fragments: _Optional[_Iterable[str]] = ...) -> None: ...
+
+class HighlightResult(_message.Message):
+    __slots__ = ("field_name", "datas")
+    FIELD_NAME_FIELD_NUMBER: _ClassVar[int]
+    DATAS_FIELD_NUMBER: _ClassVar[int]
+    field_name: str
+    datas: _containers.RepeatedCompositeFieldContainer[HighlightData]
+    def __init__(self, field_name: _Optional[str] = ..., datas: _Optional[_Iterable[_Union[HighlightData, _Mapping]]] = ...) -> None: ...
+
+class Highlighter(_message.Message):
+    __slots__ = ("type", "params")
+    TYPE_FIELD_NUMBER: _ClassVar[int]
+    PARAMS_FIELD_NUMBER: _ClassVar[int]
+    type: HighlightType
+    params: _containers.RepeatedCompositeFieldContainer[KeyValuePair]
+    def __init__(self, type: _Optional[_Union[HighlightType, str]] = ..., params: _Optional[_Iterable[_Union[KeyValuePair, _Mapping]]] = ...) -> None: ...

pymilvus/grpc_gen/common_pb2.pyi

@@ -787,6 +787,50 @@ class AddCollectionFieldRequest(_message.Message):
     schema: bytes
     def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., collectionID: _Optional[int] = ..., schema: _Optional[bytes] = ...) -> None: ...
 
+class AddCollectionFunctionRequest(_message.Message):
+    __slots__ = ("base", "db_name", "collection_name", "collectionID", "functionSchema")
+    BASE_FIELD_NUMBER: _ClassVar[int]
+    DB_NAME_FIELD_NUMBER: _ClassVar[int]
+    COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int]
+    COLLECTIONID_FIELD_NUMBER: _ClassVar[int]
+    FUNCTIONSCHEMA_FIELD_NUMBER: _ClassVar[int]
+    base: _common_pb2.MsgBase
+    db_name: str
+    collection_name: str
+    collectionID: int
+    functionSchema: _schema_pb2.FunctionSchema
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., collectionID: _Optional[int] = ..., functionSchema: _Optional[_Union[_schema_pb2.FunctionSchema, _Mapping]] = ...) -> None: ...
+
+class AlterCollectionFunctionRequest(_message.Message):
+    __slots__ = ("base", "db_name", "collection_name", "collectionID", "function_name", "functionSchema")
+    BASE_FIELD_NUMBER: _ClassVar[int]
+    DB_NAME_FIELD_NUMBER: _ClassVar[int]
+    COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int]
+    COLLECTIONID_FIELD_NUMBER: _ClassVar[int]
+    FUNCTION_NAME_FIELD_NUMBER: _ClassVar[int]
+    FUNCTIONSCHEMA_FIELD_NUMBER: _ClassVar[int]
+    base: _common_pb2.MsgBase
+    db_name: str
+    collection_name: str
+    collectionID: int
+    function_name: str
+    functionSchema: _schema_pb2.FunctionSchema
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., collectionID: _Optional[int] = ..., function_name: _Optional[str] = ..., functionSchema: _Optional[_Union[_schema_pb2.FunctionSchema, _Mapping]] = ...) -> None: ...
+
+class DropCollectionFunctionRequest(_message.Message):
+    __slots__ = ("base", "db_name", "collection_name", "collectionID", "function_name")
+    BASE_FIELD_NUMBER: _ClassVar[int]
+    DB_NAME_FIELD_NUMBER: _ClassVar[int]
+    COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int]
+    COLLECTIONID_FIELD_NUMBER: _ClassVar[int]
+    FUNCTION_NAME_FIELD_NUMBER: _ClassVar[int]
+    base: _common_pb2.MsgBase
+    db_name: str
+    collection_name: str
+    collectionID: int
+    function_name: str
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., collectionID: _Optional[int] = ..., function_name: _Optional[str] = ...) -> None: ...
+
 class UpsertRequest(_message.Message):
     __slots__ = ("base", "db_name", "collection_name", "partition_name", "fields_data", "hash_keys", "num_rows", "schema_timestamp", "partial_update", "namespace")
     BASE_FIELD_NUMBER: _ClassVar[int]
@@ -886,7 +930,7 @@ class SubSearchRequest(_message.Message):
     def __init__(self, dsl: _Optional[str] = ..., placeholder_group: _Optional[bytes] = ..., dsl_type: _Optional[_Union[_common_pb2.DslType, str]] = ..., search_params: _Optional[_Iterable[_Union[_common_pb2.KeyValuePair, _Mapping]]] = ..., nq: _Optional[int] = ..., expr_template_values: _Optional[_Mapping[str, _schema_pb2.TemplateValue]] = ..., namespace: _Optional[str] = ...) -> None: ...
 
 class SearchRequest(_message.Message):
-    __slots__ = ("base", "db_name", "collection_name", "partition_names", "dsl", "placeholder_group", "dsl_type", "output_fields", "search_params", "travel_timestamp", "guarantee_timestamp", "nq", "not_return_all_meta", "consistency_level", "use_default_consistency", "search_by_primary_keys", "sub_reqs", "expr_template_values", "function_score", "namespace")
+    __slots__ = ("base", "db_name", "collection_name", "partition_names", "dsl", "placeholder_group", "dsl_type", "output_fields", "search_params", "travel_timestamp", "guarantee_timestamp", "nq", "not_return_all_meta", "consistency_level", "use_default_consistency", "search_by_primary_keys", "sub_reqs", "expr_template_values", "function_score", "namespace", "highlighter")
     class ExprTemplateValuesEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -914,6 +958,7 @@ class SearchRequest(_message.Message):
     EXPR_TEMPLATE_VALUES_FIELD_NUMBER: _ClassVar[int]
     FUNCTION_SCORE_FIELD_NUMBER: _ClassVar[int]
     NAMESPACE_FIELD_NUMBER: _ClassVar[int]
+    HIGHLIGHTER_FIELD_NUMBER: _ClassVar[int]
     base: _common_pb2.MsgBase
     db_name: str
     collection_name: str
@@ -934,7 +979,8 @@ class SearchRequest(_message.Message):
     expr_template_values: _containers.MessageMap[str, _schema_pb2.TemplateValue]
     function_score: _schema_pb2.FunctionScore
     namespace: str
-    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., partition_names: _Optional[_Iterable[str]] = ..., dsl: _Optional[str] = ..., placeholder_group: _Optional[bytes] = ..., dsl_type: _Optional[_Union[_common_pb2.DslType, str]] = ..., output_fields: _Optional[_Iterable[str]] = ..., search_params: _Optional[_Iterable[_Union[_common_pb2.KeyValuePair, _Mapping]]] = ..., travel_timestamp: _Optional[int] = ..., guarantee_timestamp: _Optional[int] = ..., nq: _Optional[int] = ..., not_return_all_meta: bool = ..., consistency_level: _Optional[_Union[_common_pb2.ConsistencyLevel, str]] = ..., use_default_consistency: bool = ..., search_by_primary_keys: bool = ..., sub_reqs: _Optional[_Iterable[_Union[SubSearchRequest, _Mapping]]] = ..., expr_template_values: _Optional[_Mapping[str, _schema_pb2.TemplateValue]] = ..., function_score: _Optional[_Union[_schema_pb2.FunctionScore, _Mapping]] = ..., namespace: _Optional[str] = ...) -> None: ...
+    highlighter: _common_pb2.Highlighter
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., partition_names: _Optional[_Iterable[str]] = ..., dsl: _Optional[str] = ..., placeholder_group: _Optional[bytes] = ..., dsl_type: _Optional[_Union[_common_pb2.DslType, str]] = ..., output_fields: _Optional[_Iterable[str]] = ..., search_params: _Optional[_Iterable[_Union[_common_pb2.KeyValuePair, _Mapping]]] = ..., travel_timestamp: _Optional[int] = ..., guarantee_timestamp: _Optional[int] = ..., nq: _Optional[int] = ..., not_return_all_meta: bool = ..., consistency_level: _Optional[_Union[_common_pb2.ConsistencyLevel, str]] = ..., use_default_consistency: bool = ..., search_by_primary_keys: bool = ..., sub_reqs: _Optional[_Iterable[_Union[SubSearchRequest, _Mapping]]] = ..., expr_template_values: _Optional[_Mapping[str, _schema_pb2.TemplateValue]] = ..., function_score: _Optional[_Union[_schema_pb2.FunctionScore, _Mapping]] = ..., namespace: _Optional[str] = ..., highlighter: _Optional[_Union[_common_pb2.Highlighter, _Mapping]] = ...) -> None: ...
 
 class Hits(_message.Message):
     __slots__ = ("IDs", "row_data", "scores")
@@ -2432,28 +2478,24 @@ class RunAnalyzerResponse(_message.Message):
     def __init__(self, status: _Optional[_Union[_common_pb2.Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[AnalyzerResult, _Mapping]]] = ...) -> None: ...
 
 class FileResourceInfo(_message.Message):
-    __slots__ = ("id", "name", "path", "type")
+    __slots__ = ("id", "name", "path")
     ID_FIELD_NUMBER: _ClassVar[int]
     NAME_FIELD_NUMBER: _ClassVar[int]
     PATH_FIELD_NUMBER: _ClassVar[int]
-    TYPE_FIELD_NUMBER: _ClassVar[int]
     id: int
     name: str
     path: str
-    type: _common_pb2.FileResourceType
-    def __init__(self, id: _Optional[int] = ..., name: _Optional[str] = ..., path: _Optional[str] = ..., type: _Optional[_Union[_common_pb2.FileResourceType, str]] = ...) -> None: ...
+    def __init__(self, id: _Optional[int] = ..., name: _Optional[str] = ..., path: _Optional[str] = ...) -> None: ...
 
 class AddFileResourceRequest(_message.Message):
-    __slots__ = ("base", "name", "path", "type")
+    __slots__ = ("base", "name", "path")
     BASE_FIELD_NUMBER: _ClassVar[int]
     NAME_FIELD_NUMBER: _ClassVar[int]
     PATH_FIELD_NUMBER: _ClassVar[int]
-    TYPE_FIELD_NUMBER: _ClassVar[int]
     base: _common_pb2.MsgBase
     name: str
     path: str
-    type: _common_pb2.FileResourceType
-    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., name: _Optional[str] = ..., path: _Optional[str] = ..., type: _Optional[_Union[_common_pb2.FileResourceType, str]] = ...) -> None: ...
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., name: _Optional[str] = ..., path: _Optional[str] = ...) -> None: ...
 
 class RemoveFileResourceRequest(_message.Message):
     __slots__ = ("base", "name")
@@ -2626,16 +2668,18 @@ class UpdateReplicateConfigurationRequest(_message.Message):
     def __init__(self, replicate_configuration: _Optional[_Union[_common_pb2.ReplicateConfiguration, _Mapping]] = ...) -> None: ...
 
 class GetReplicateInfoRequest(_message.Message):
-    __slots__ = ("source_cluster_id",)
+    __slots__ = ("source_cluster_id", "target_pchannel")
     SOURCE_CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    TARGET_PCHANNEL_FIELD_NUMBER: _ClassVar[int]
     source_cluster_id: str
-    def __init__(self, source_cluster_id: _Optional[str] = ...) -> None: ...
+    target_pchannel: str
+    def __init__(self, source_cluster_id: _Optional[str] = ..., target_pchannel: _Optional[str] = ...) -> None: ...
 
 class GetReplicateInfoResponse(_message.Message):
-    __slots__ = ("checkpoints",)
-    CHECKPOINTS_FIELD_NUMBER: _ClassVar[int]
-    checkpoints: _containers.RepeatedCompositeFieldContainer[_common_pb2.ReplicateCheckpoint]
-    def __init__(self, checkpoints: _Optional[_Iterable[_Union[_common_pb2.ReplicateCheckpoint, _Mapping]]] = ...) -> None: ...
+    __slots__ = ("checkpoint",)
+    CHECKPOINT_FIELD_NUMBER: _ClassVar[int]
+    checkpoint: _common_pb2.ReplicateCheckpoint
+    def __init__(self, checkpoint: _Optional[_Union[_common_pb2.ReplicateCheckpoint, _Mapping]] = ...) -> None: ...
 
 class ReplicateMessage(_message.Message):
     __slots__ = ("source_cluster_id", "message")