diff --git a/deployment/library/indexer_other_file_types_template.json b/deployment/library/indexer_other_file_types_template.json new file mode 100644 index 00000000..c3df3a84 --- /dev/null +++ b/deployment/library/indexer_other_file_types_template.json @@ -0,0 +1,32 @@ +{ + "name": "${indexer_name}", + "description": "${indexer_description}", + "dataSourceName": "${datasource_name}", + "skillsetName": "${skillset_name}", + "targetIndexName": "${index_name}", + "disabled": false, + "schedule": null, + "parameters": { + "batchSize": 1, + "maxFailedItems": 0, + "maxFailedItemsPerBatch": 0, + "base64EncodeKeys": null, + "configuration": { + "parsingMode": "default", + "excludedFileNameExtensions": ".pdf", + "indexedFileNameExtensions": ".docx, .doc, .docm, .xlsx, .xls, .xlsm, .pptx, .ppt, .pptm, .msg, .txt, .csv, .eml, .epub, .gz, .xml, .json, .html, .kml, .odt, .ods, .odp, .rtf, .zip, .jpeg, .png, .bmp, .tif", + "failOnUnsupportedContentType": false, + "failOnUnprocessableDocument": false, + "indexStorageMetadataOnlyForOversizedDocuments": false, + "firstLineContainsHeaders": true, + "dataToExtract": "contentAndMetadata", + "imageAction": "generateNormalizedImages", + "allowSkillsetToReadFileData": false, + "executionEnvironment": "private" + } + }, + "fieldMappings": [], + "outputFieldMappings": [], + "cache": null, + "encryptionKey": null +} diff --git a/deployment/library/skillset_other_file_types_template.json b/deployment/library/skillset_other_file_types_template.json new file mode 100644 index 00000000..304dd872 --- /dev/null +++ b/deployment/library/skillset_other_file_types_template.json @@ -0,0 +1,86 @@ +{ + "name": "${skillset_name}", + "description": "${skillset_name}", + "skills": [ + { + "@odata.type": "#Microsoft.Skills.Text.SplitSkill", + "name": "microsoft_text_split_skill", + "description": "Split skill to chunk documents", + "context": "/document", + "defaultLanguageCode": "en", + "textSplitMode": "pages", + "maximumPageLength": 2000, + "pageOverlapLength": 500, + "maximumPagesToTake": 0, + "unit": "characters", + "inputs": [ + { + "name": "text", + "source": "/document/content", + "inputs": [] + } + ], + "outputs": [ + { + "name": "textItems", + "targetName": "pages" + } + ] + }, + { + "@odata.type": "#Microsoft.Skills.Text.AzureOpenAIEmbeddingSkill", + "name": "azure_openai_text_embedding_skill", + "description": "Skill to generate embeddings via Azure OpenAI", + "context": "/document/pages/*", + "resourceUri": "${azureOpenAI_endpoint}", + "apiKey": null, + "deploymentId": "${azureOpenAI_text_deployment_id}", + "dimensions": 1536, + "modelName": "${azureOpenAI_text_model_name}", + "inputs": [ + { + "name": "text", + "source": "/document/pages/*" + } + ], + "outputs": [ + { + "name": "embedding", + "targetName": "text_vector" + } + ], + "authIdentity": null + } + ], + "cognitiveServices": { + "@odata.type": "#Microsoft.Azure.Search.AIServicesByIdentity", + "description": null, + "subdomainUrl": "${cognitiveServices_multiService_endpoint}", + "identity": null + }, + "indexProjections": { + "selectors": [ + { + "targetIndexName": "${index_name}", + "parentKeyFieldName": "parent_id", + "sourceContext": "/document/pages/*", + "mappings": [ + { + "name": "content", + "source": "/document/pages/*", + "inputs": [] + }, + { + "name": "embedding", + "source": "/document/pages/*/text_vector", + "inputs": [] + } + ] + } + ], + "parameters": { + "projectionMode": "skipIndexingParentDocuments" + } + }, + "encryptionKey": null +} diff --git a/deployment/terraform/infra/modules/aisearch/aisearch-indexer-other-file-types.tf b/deployment/terraform/infra/modules/aisearch/aisearch-indexer-other-file-types.tf new file mode 100644 index 00000000..6301f5df --- /dev/null +++ b/deployment/terraform/infra/modules/aisearch/aisearch-indexer-other-file-types.tf @@ -0,0 +1,46 @@ + +locals { + indexer_other_file_types_config_file = "indexer_other_file_types_config.json" +} + +resource "local_file" "indexer_config_other_file_types" { + content = templatefile("${path.module}/../../../../library/indexer_other_file_types_template.json", { + datasource_name = var.search_service_datasource_name + index_name = var.search_service_index_name + indexer_name = var.search_service_indexer_other_file_types_name + indexer_description = "Indexer for auto indexing documents with ${var.search_service_skillset_other_file_types_name}" + skillset_name = var.search_service_skillset_other_file_types_name + }) + filename = "${path.module}/${local.indexer_other_file_types_config_file}" +} + +# wait for permissions to propogate +resource "time_sleep" "wait_permissions_to_propagate_other_file_types" { + depends_on = [azurerm_role_assignment.storage_blob_data_to_search_service, azurerm_role_assignment.knowledgestore_blob_data_to_search_service] + create_duration = "2m" +} + +resource "null_resource" "create_indexer_other_file_types" { + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell", "-Command"] : [] + command = <