Issues with adding a new custom field in searchmanager.py

[rajib2904]

All,
I have added one addional field in the index.But while creating index I am getting following error. Can you please help me what went wrong here?

azure.core.exceptions.HttpResponseError: () The request is invalid. Details: The property 'skey' does not exist on type 'search.documentFields' or is not present in the API version '2025-08-01-preview

Please find below modified code
async with self.search_info.create_search_client() as search_client:
for batch_index, batch in enumerate(section_batches):
documents = []
for section_index, section in enumerate(batch):
image_fields = {}
if self.search_images:
image_fields = {
"images": [
{
"url": image.url,
"description": image.description,
"boundingbox": image.bbox,
"embedding": image.embedding,
}
for image in section.chunk.images
]
}
document = {
# "id": f"{section.content.filename_to_id()}-page-{section_index + batch_index * MAX_BATCH_SIZE}",
"id": f"{fileKey}-p{section_index + batch_index * MAX_BATCH_SIZE}",
"skey": f"{fileKey}",
"content": section.chunk.text,
"category": section.category,

                    "sourcepage": BlobManager.sourcepage_from_file_page(
                        filename=section.content.filename(), page=section.chunk.page_num
                    ),
                    "sourcefile": section.content.filename(),
                    **image_fields,
                    **section.content.acls,
                }
                documents.append(document)
            if url:
                for document in documents:
                    document["storageUrl"] = url
            if self.embeddings:
                if self.field_name_embedding is None:
                    raise ValueError("Embedding field name must be set")
                embeddings = await self.embeddings.create_embeddings(
                    texts=[section.chunk.text for section in batch]
                )
                for i, document in enumerate(documents):
                    document[self.field_name_embedding] = embeddings[i]
            logger.info(
                "Uploading batch %d with %d sections to search index '%s'",
                batch_index + 1,
                len(documents),
                self.search_info.index_name,
            )

[JasonHaley]

Does the skey field exist in your index? I don't use the vision stuff, so not 100% sure but there is no skey field in my indexes.

Probably need to add it somewhere in the create_index() method in searchmanager.py. I have added several fields to my index for additional filtering I need to do - and that is the location I've added them to get the code to create the index if it isn't there. Otherwise you could modify the Azure AI Search index json in Azure too.

[rajib2904]

Does the skey field exist in your index? I don't use the vision stuff, so not 100% sure but there is no skey field in my indexes.

Probably need to add it somewhere in the create_index() method in searchmanager.py. I have added several fields to my index for additional filtering I need to do - and that is the location I've added them to get the code to create the index if it isn't there. Otherwise you could modify the Azure AI Search index json in Azure too.

I add this field in create_index() function.Please see below
async def create_index(self):
logger.info("Checking whether search index %s exists...", self.search_info.index_name)

    async with self.search_info.create_search_index_client() as search_index_client:
        embedding_field = None
        images_field = None
        text_vector_search_profile = None
        text_vector_algorithm = None
        text_vector_compression = None
        image_vector_search_profile = None
        image_vector_algorithm = None

        if self.embeddings:
            if self.embedding_dimensions is None:
                raise ValueError(
                    "Embedding dimensions must be set in order to add an embedding field to the search index"
                )
            if self.field_name_embedding is None:
                raise ValueError(
                    "Embedding field must be set in order to add an embedding field to the search index"
                )

            text_vectorizer = None
            if isinstance(self.embeddings, AzureOpenAIEmbeddingService):
                text_vectorizer = AzureOpenAIVectorizer(
                    vectorizer_name=f"{self.embeddings.open_ai_model_name}-vectorizer",
                    parameters=AzureOpenAIVectorizerParameters(
                        resource_url=self.embeddings.open_ai_endpoint,
                        deployment_name=self.embeddings.open_ai_deployment,
                        model_name=self.embeddings.open_ai_model_name,
                    ),
                )

            text_vector_algorithm = HnswAlgorithmConfiguration(
                name="hnsw_config",
                parameters=HnswParameters(metric="cosine"),
            )
            text_vector_compression = BinaryQuantizationCompression(
                compression_name=f"{self.field_name_embedding}-compression",
                truncation_dimension=1024,  # should this be a parameter? maybe not yet?
                rescoring_options=RescoringOptions(
                    enable_rescoring=True,
                    default_oversampling=10,
                    rescore_storage_method=VectorSearchCompressionRescoreStorageMethod.PRESERVE_ORIGINALS,
                ),
                # Explicitly set deprecated parameters to None
                rerank_with_original_vectors=None,
                default_oversampling=None,
            )
            text_vector_search_profile = VectorSearchProfile(
                name=f"{self.field_name_embedding}-profile",
                algorithm_configuration_name=text_vector_algorithm.name,
                compression_name=text_vector_compression.compression_name,
                **({"vectorizer_name": text_vectorizer.vectorizer_name if text_vectorizer else None}),
            )

            embedding_field = SearchField(
                name=self.field_name_embedding,
                type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
                hidden=True,
                searchable=True,
                filterable=False,
                sortable=False,
                facetable=False,
                vector_search_dimensions=self.embedding_dimensions,
                vector_search_profile_name=f"{self.field_name_embedding}-profile",
                stored=False,
            )

        if self.search_images:
            if not self.search_info.azure_vision_endpoint:
                raise ValueError("Azure AI Vision endpoint must be provided to use image embeddings")
            image_vector_algorithm = HnswAlgorithmConfiguration(
                name="images_hnsw_config",
                parameters=HnswParameters(metric="cosine"),
            )

            # Create the AI Vision vectorizer for image embeddings
            image_vectorizer = AIServicesVisionVectorizer(
                vectorizer_name="images-vision-vectorizer",
                ai_services_vision_parameters=AIServicesVisionParameters(
                    resource_uri=self.search_info.azure_vision_endpoint,
                    model_version="2023-04-15",
                ),
            )

            image_vector_search_profile = VectorSearchProfile(
                name="images_embedding_profile",
                algorithm_configuration_name=image_vector_algorithm.name,
                vectorizer_name=image_vectorizer.vectorizer_name,
            )
            images_field = SearchField(
                name="images",
                type=SearchFieldDataType.Collection(SearchFieldDataType.ComplexType),
                fields=[
                    SearchField(
                        name="embedding",
                        type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
                        searchable=True,
                        stored=False,
                        vector_search_dimensions=1024,
                        vector_search_profile_name=image_vector_search_profile.name,
                    ),
                    SearchField(
                        name="url",
                        type=SearchFieldDataType.String,
                        searchable=False,
                        filterable=True,
                        sortable=False,
                        facetable=True,
                    ),
                    SearchField(
                        name="description",
                        type=SearchFieldDataType.String,
                        searchable=True,
                        filterable=False,
                        sortable=False,
                        facetable=False,
                    ),
                    SearchField(
                        name="boundingbox",
                        type=SearchFieldDataType.Collection(SearchFieldDataType.Double),
                        searchable=False,
                        filterable=False,
                        sortable=False,
                        facetable=False,
                    ),
                ],
            )

        if self.search_info.index_name not in [name async for name in search_index_client.list_index_names()]:
            logger.info("Creating new search index %s", self.search_info.index_name)
            fields = [
                (
                    SimpleField(name="id", type="Edm.String", key=True)
                    if not self.use_int_vectorization
                    else SearchField(
                        name="id",
                        type="Edm.String",
                        key=True,
                        sortable=True,
                        # filterable=True,
                        facetable=True,
                        analyzer_name="keyword",
                    )
                ),
                **SimpleField(name="skey", type="Edm.String", filterable=True, facetable=True),**
                SearchableField(
                    name="content",
                    type="Edm.String",
                    analyzer_name=self.search_analyzer_name,
                ),
                SimpleField(name="category", type="Edm.String", filterable=True, facetable=True),

                SimpleField(
                    name="sourcepage",
                    type="Edm.String",
                    filterable=True,
                    facetable=True,
                ),
                SimpleField(
                    name="sourcefile",
                    type="Edm.String",
                    filterable=True,
                    facetable=True,
                ),
                SimpleField(
                    name="storageUrl",
                    type="Edm.String",
                    filterable=True,
                    facetable=False,
                ),
            ]
            if self.use_acls:
                fields.append(
                    SimpleField(
                        name="oids",
                        type=SearchFieldDataType.Collection(SearchFieldDataType.String),
                        filterable=True,
                    )
                )
                fields.append(
                    SimpleField(
                        name="groups",
                        type=SearchFieldDataType.Collection(SearchFieldDataType.String),
                        filterable=True,
                    )
                )

            if self.use_int_vectorization:
                logger.info("Including parent_id field for integrated vectorization support in new index")
                fields.append(SearchableField(name="parent_id", type="Edm.String", filterable=True))

            vectorizers: list[VectorSearchVectorizer] = []
            vector_search_profiles = []
            vector_algorithms: list[VectorSearchAlgorithmConfiguration] = []
            vector_compressions: list[VectorSearchCompression] = []
            if embedding_field:
                logger.info("Including %s field for text vectors in new index", embedding_field.name)
                fields.append(embedding_field)
                if text_vectorizer is not None:
                    vectorizers.append(text_vectorizer)
                if (
                    text_vector_search_profile is None
                    or text_vector_algorithm is None
                    or text_vector_compression is None
                ):
                    raise ValueError("Text vector search profile, algorithm and compression must be set")
                vector_search_profiles.append(text_vector_search_profile)
                vector_algorithms.append(text_vector_algorithm)
                vector_compressions.append(text_vector_compression)

            if images_field:
                logger.info("Including %s field for image descriptions and vectors in new index", images_field.name)
                fields.append(images_field)
                if image_vector_search_profile is None or image_vector_algorithm is None:
                    raise ValueError("Image search profile and algorithm must be set")
                vector_search_profiles.append(image_vector_search_profile)
                vector_algorithms.append(image_vector_algorithm)
                # Add image vectorizer to vectorizers list
                vectorizers.append(image_vectorizer)

            index = SearchIndex(
                name=self.search_info.index_name,
                fields=fields,
                semantic_search=SemanticSearch(
                    default_configuration_name="default",
                    configurations=[
                        SemanticConfiguration(
                            name="default",
                            prioritized_fields=SemanticPrioritizedFields(
                                title_field=SemanticField(field_name="sourcepage"),
                                content_fields=[SemanticField(field_name="content")],
                            ),
                        )
                    ],
                ),
                vector_search=VectorSearch(
                    profiles=vector_search_profiles,
                    algorithms=vector_algorithms,
                    compressions=vector_compressions,
                    vectorizers=vectorizers,
                ),
            )

            await search_index_client.create_index(index)
        else:
            logger.info("Search index %s already exists", self.search_info.index_name)
            existing_index = await search_index_client.get_index(self.search_info.index_name)
            if not any(field.name == "storageUrl" for field in existing_index.fields):
                logger.info("Adding storageUrl field to index %s", self.search_info.index_name)
                existing_index.fields.append(
                    SimpleField(
                        name="storageUrl",
                        type="Edm.String",
                        filterable=True,
                        facetable=False,
                    ),
                )
                await search_index_client.create_or_update_index(existing_index)

            if embedding_field and not any(
                field.name == self.field_name_embedding for field in existing_index.fields
            ):
                logger.info("Adding %s field for text embeddings", self.field_name_embedding)
                embedding_field.stored = True
                existing_index.fields.append(embedding_field)
                if existing_index.vector_search is None:
                    raise ValueError("Vector search is not enabled for the existing index")
                if text_vectorizer is not None:
                    if existing_index.vector_search.vectorizers is None:
                        existing_index.vector_search.vectorizers = []
                    existing_index.vector_search.vectorizers.append(text_vectorizer)
                if (
                    text_vector_search_profile is None
                    or text_vector_algorithm is None
                    or text_vector_compression is None
                ):
                    raise ValueError("Text vector search profile, algorithm and compression must be set")
                if existing_index.vector_search.profiles is None:
                    existing_index.vector_search.profiles = []
                existing_index.vector_search.profiles.append(text_vector_search_profile)
                if existing_index.vector_search.algorithms is None:
                    existing_index.vector_search.algorithms = []
                # existing_index.vector_search.algorithms.append(text_vector_algorithm)
                if existing_index.vector_search.compressions is None:
                    existing_index.vector_search.compressions = []
                existing_index.vector_search.compressions.append(text_vector_compression)
                await search_index_client.create_or_update_index(existing_index)

            if (
                images_field
                and images_field.fields
                and not any(field.name == "images" for field in existing_index.fields)
            ):
                logger.info("Adding %s field for image embeddings", images_field.name)
                images_field.fields[0].stored = True
                existing_index.fields.append(images_field)
                if image_vector_search_profile is None or image_vector_algorithm is None:
                    raise ValueError("Image vector search profile and algorithm must be set")
                if existing_index.vector_search is None:
                    raise ValueError("Image vector search is not enabled for the existing index")
                if existing_index.vector_search.profiles is None:
                    existing_index.vector_search.profiles = []
                existing_index.vector_search.profiles.append(image_vector_search_profile)
                if existing_index.vector_search.algorithms is None:
                    existing_index.vector_search.algorithms = []
                existing_index.vector_search.algorithms.append(image_vector_algorithm)
                if existing_index.vector_search.vectorizers is None:
                    existing_index.vector_search.vectorizers = []
                existing_index.vector_search.vectorizers.append(image_vectorizer)
                await search_index_client.create_or_update_index(existing_index)

            if existing_index.semantic_search:
                if not existing_index.semantic_search.default_configuration_name:
                    logger.info("Adding default semantic configuration to index %s", self.search_info.index_name)
                    existing_index.semantic_search.default_configuration_name = "default"

                if existing_index.semantic_search.configurations:
                    existing_semantic_config = existing_index.semantic_search.configurations[0]
                    if (
                        existing_semantic_config.prioritized_fields
                        and existing_semantic_config.prioritized_fields.title_field
                        and not existing_semantic_config.prioritized_fields.title_field.field_name == "sourcepage"
                    ):
                        logger.info("Updating semantic configuration for index %s", self.search_info.index_name)
                        existing_semantic_config.prioritized_fields.title_field = SemanticField(
                            field_name="sourcepage"
                        )

            if existing_index.vector_search is not None and (
                existing_index.vector_search.vectorizers is None
                or len(existing_index.vector_search.vectorizers) == 0
            ):
                if self.embeddings is not None and isinstance(self.embeddings, AzureOpenAIEmbeddingService):
                    logger.info("Adding vectorizer to search index %s", self.search_info.index_name)
                    existing_index.vector_search.vectorizers = [
                        AzureOpenAIVectorizer(
                            vectorizer_name=f"{self.search_info.index_name}-vectorizer",
                            parameters=AzureOpenAIVectorizerParameters(
                                resource_url=self.embeddings.open_ai_endpoint,
                                deployment_name=self.embeddings.open_ai_deployment,
                                model_name=self.embeddings.open_ai_model_name,
                            ),
                        )
                    ]
                    await search_index_client.create_or_update_index(existing_index)

                else:
                    logger.info(
                        "Can't add vectorizer to search index %s since no Azure OpenAI embeddings service is defined",
                        self.search_info,
                    )

    if self.search_info.use_agentic_retrieval and self.search_info.agent_name:
        await self.create_agent()

[pamelafox]

Are you creating the new index from scratch? Make sure that it's actually going down the path of creating the new index. That code is admittedly a bit confusing.