Provide support for indexing Office and other file types

deployment/library/indexer_other_file_types_template.json

@@ -0,0 +1,32 @@
+{
+  "name": "${indexer_name}",
+  "description": "${indexer_description}",
+  "dataSourceName": "${datasource_name}",
+  "skillsetName": "${skillset_name}",
+  "targetIndexName": "${index_name}",
+  "disabled": false,
+  "schedule": null,
+  "parameters": {
+    "batchSize": 1,
+    "maxFailedItems": 0,
+    "maxFailedItemsPerBatch": 0,
+    "base64EncodeKeys": null,
+    "configuration": {
+      "parsingMode": "default",
+      "excludedFileNameExtensions": ".pdf",
+      "indexedFileNameExtensions": ".docx, .doc, .docm, .xlsx, .xls, .xlsm, .pptx, .ppt, .pptm, .msg, .txt, .csv, .eml, .epub, .gz, .xml, .json, .html, .kml, .odt, .ods, .odp, .rtf, .zip, .jpeg, .png, .bmp, .tif",
+      "failOnUnsupportedContentType": false,
+      "failOnUnprocessableDocument": false,
+      "indexStorageMetadataOnlyForOversizedDocuments": false,
+      "firstLineContainsHeaders": true,
+      "dataToExtract": "contentAndMetadata",
+      "imageAction": "generateNormalizedImages",
+      "allowSkillsetToReadFileData": false,
+      "executionEnvironment": "private"
+    }
+  },
+  "fieldMappings": [],
+  "outputFieldMappings": [],
+  "cache": null,
+  "encryptionKey": null
+}

deployment/library/skillset_other_file_types_template.json

@@ -0,0 +1,86 @@
+{
+  "name": "${skillset_name}",
+  "description": "${skillset_name}",
+  "skills": [
+    {
+      "@odata.type": "#Microsoft.Skills.Text.SplitSkill",
+      "name": "microsoft_text_split_skill",
+      "description": "Split skill to chunk documents",
+      "context": "/document",
+      "defaultLanguageCode": "en",
+      "textSplitMode": "pages",
+      "maximumPageLength": 2000,
+      "pageOverlapLength": 500,
+      "maximumPagesToTake": 0,
+      "unit": "characters",
+      "inputs": [
+        {
+          "name": "text",
+          "source": "/document/content",
+          "inputs": []
+        }
+      ],
+      "outputs": [
+        {
+          "name": "textItems",
+          "targetName": "pages"
+        }
+      ]
+    },
+    {
+      "@odata.type": "#Microsoft.Skills.Text.AzureOpenAIEmbeddingSkill",
+      "name": "azure_openai_text_embedding_skill",
+      "description": "Skill to generate embeddings via Azure OpenAI",
+      "context": "/document/pages/*",
+      "resourceUri": "${azureOpenAI_endpoint}",
+      "apiKey": null,
+      "deploymentId": "${azureOpenAI_text_deployment_id}",
+      "dimensions": 1536,
+      "modelName": "${azureOpenAI_text_model_name}",
+      "inputs": [
+        {
+          "name": "text",
+          "source": "/document/pages/*"
+        }
+      ],
+      "outputs": [
+        {
+          "name": "embedding",
+          "targetName": "text_vector"
+        }
+      ],
+      "authIdentity": null
+    }
+  ],
+  "cognitiveServices": {
+    "@odata.type": "#Microsoft.Azure.Search.AIServicesByIdentity",
+    "description": null,
+    "subdomainUrl": "${cognitiveServices_multiService_endpoint}",
+    "identity": null
+  },
+  "indexProjections": {
+    "selectors": [
+      {
+        "targetIndexName": "${index_name}",
+        "parentKeyFieldName": "parent_id",
+        "sourceContext": "/document/pages/*",
+        "mappings": [
+          {
+            "name": "content",
+            "source": "/document/pages/*",
+            "inputs": []
+          },
+          {
+            "name": "embedding",
+            "source": "/document/pages/*/text_vector",
+            "inputs": []
+          }
+        ]
+      }
+    ],
+    "parameters": {
+      "projectionMode": "skipIndexingParentDocuments"
+    }
+  },
+  "encryptionKey": null
+}

deployment/terraform/infra/modules/aisearch/aisearch-indexer-other-file-types.tf

@@ -0,0 +1,46 @@
+
+locals {
+  indexer_other_file_types_config_file = "indexer_other_file_types_config.json"
+}
+
+resource "local_file" "indexer_config_other_file_types" {
+  content = templatefile("${path.module}/../../../../library/indexer_other_file_types_template.json", {
+    datasource_name     = var.search_service_datasource_name
+    index_name          = var.search_service_index_name
+    indexer_name        = var.search_service_indexer_other_file_types_name
+    indexer_description = "Indexer for auto indexing documents with ${var.search_service_skillset_other_file_types_name}"
+    skillset_name       = var.search_service_skillset_other_file_types_name
+  })
+  filename = "${path.module}/${local.indexer_other_file_types_config_file}"
+}
+
+# wait for permissions to propogate
+resource "time_sleep" "wait_permissions_to_propagate_other_file_types" {
+  depends_on      = [azurerm_role_assignment.storage_blob_data_to_search_service, azurerm_role_assignment.knowledgestore_blob_data_to_search_service]
+  create_duration = "2m"
+}
+
+resource "null_resource" "create_indexer_other_file_types" {
+  provisioner "local-exec" {
+    interpreter = local.is_windows ? ["PowerShell", "-Command"] : []
+    command     = <<EOT
+      ${local.get_access_token_command}
+      az rest --method PUT ${local.line_separator}
+        --url ${local.escape_char}"https://${var.search_service_name}.search.windows.net/indexers${local.escape_char}(${local.escape_char}'${var.search_service_indexer_other_file_types_name}${local.escape_char}'${local.escape_char})?api-version=2024-05-01-preview${local.escape_char}" ${local.line_separator}
+        --headers ${local.escape_char}"Content-Type=application/json${local.escape_char}" ${local.escape_char}"Authorization=Bearer $ACCESS_TOKEN${local.escape_char}" ${local.line_separator}
+        --body ${local.escape_char}@${path.module}${local.path_separator}${local.indexer_other_file_types_config_file}
+      ${format(local.delete_file_command, local.indexer_other_file_types_config_file)}
+    EOT
+  }
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+  depends_on = [
+    azurerm_search_service.search_service,
+    local_file.indexer_config_other_file_types,
+    null_resource.create_datasource,
+    null_resource.create_index,
+    null_resource.create_skillset_other_file_types,
+    time_sleep.wait_permissions_to_propagate_other_file_types
+  ]
+}

deployment/terraform/infra/modules/aisearch/aisearch-skillset-other-file-types.tf

@@ -0,0 +1,42 @@
+locals {
+  skillset_other_file_types_config_file = "skillset_config_other_file_types.json"
+}
+
+resource "local_file" "skillset_other_file_types_config" {
+  content = templatefile("${path.module}/../../../../library/skillset_other_file_types_template.json", {
+    index_name                              = var.search_service_index_name
+    skillset_name                           = var.search_service_skillset_other_file_types_name
+    azureOpenAI_endpoint                    = var.azure_openai_endpoint
+    azureOpenAI_text_deployment_id          = var.azure_openai_text_deployment_id
+    azureOpenAI_text_model_name             = var.azure_openai_text_model_name
+    cognitiveServices_multiService_endpoint = var.cognitive_services_endpoint
+  })
+  filename = "${path.module}/${local.skillset_other_file_types_config_file}"
+}
+
+
+resource "null_resource" "create_skillset_other_file_types" {
+  provisioner "local-exec" {
+    interpreter = local.is_windows ? ["PowerShell", "-Command"] : []
+    command     = <<EOT
+      ${local.get_access_token_command}
+      az rest --method PUT ${local.line_separator}
+        --url ${local.escape_char}"https://${var.search_service_name}.search.windows.net/skillsets${local.escape_char}(${local.escape_char}'${var.search_service_skillset_other_file_types_name}${local.escape_char}'${local.escape_char})?api-version=2024-11-01-preview${local.escape_char}" ${local.line_separator}
+        --headers ${local.escape_char}"Content-Type=application/json${local.escape_char}" ${local.escape_char}"Authorization=Bearer $ACCESS_TOKEN${local.escape_char}" ${local.line_separator}
+        --body ${local.escape_char}@${path.module}${local.path_separator}${local.skillset_other_file_types_config_file}
+      ${format(local.delete_file_command, local.skillset_other_file_types_config_file)}
+    EOT
+  }
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+  depends_on = [
+    azurerm_search_service.search_service,
+    local_file.skillset_other_file_types_config,
+    null_resource.create_index,
+    null_resource.create_datasource,
+    null_resource.create_skillset,
+    null_resource.create_indexer,
+    azapi_update_resource.function_azure_search_private_endpoint_approver
+  ]
+}

deployment/terraform/infra/modules/aisearch/variables.tf

@@ -126,14 +126,26 @@ variable "search_service_indexer_name" {
   sensitive   = false
 }
 
+variable "search_service_indexer_other_file_types_name" {
+  description = "Specifies index name for other file types (.docx, .xlsx, .pptx, etc.)."
+  type        = string
+  sensitive   = false
+}
+
 variable "openai_account_id" {
   description = "Specifies the id for the AOAI account."
   type        = string
   sensitive   = false
 }
 
 variable "search_service_skillset_name" {
-  description = "Specifies index name."
+  description = "Specifies skillset name."
+  type        = string
+  sensitive   = false
+}
+
+variable "search_service_skillset_other_file_types_name" {
+  description = "Specifies skillset for other file types name."
   type        = string
   sensitive   = false
 }

deployment/terraform/infra/service.tf

@@ -211,40 +211,42 @@ module "backend_webapp" {
 }
 
 module "aisearch" {
-  source                                = "./modules/aisearch"
-  location                              = var.search_service_location != "" ? var.search_service_location : var.location
-  tags                                  = local.tags
-  resource_group_name                   = azurerm_resource_group.resource_group.name
-  log_analytics_workspace_id            = var.log_analytics_workspace_id
-  search_service_name                   = var.search_service_name != "" ? var.search_service_name : "${local.abbrs.searchServices}${local.resourceToken}"
-  search_service_sku                    = var.search_service_sku
-  semantic_search_sku                   = var.semantic_search_sku
-  search_service_partition_count        = var.search_service_partition_count
-  search_service_replica_count          = var.search_service_replica_count
-  storage_account_id                    = module.storage.storage_account_id
-  storage_container_name_content        = var.storage_container_name_content
-  search_service_datasource_name        = var.search_service_datasource_name != "" ? var.search_service_datasource_name : "${local.abbrs.searchServices}ds-${local.resourceToken}"
-  search_service_index_name             = var.search_service_index_name != "" ? var.search_service_index_name : "${local.abbrs.searchServices}ind-${local.resourceToken}"
-  search_service_indexer_name           = var.search_service_indexer_name != "" ? var.search_service_indexer_name : "${local.abbrs.searchServices}inder-${local.resourceToken}"
-  search_service_skillset_name          = var.search_service_skillset_name != "" ? var.search_service_skillset_name : "${local.abbrs.searchServices}skl-${local.resourceToken}"
-  azure_openai_endpoint                 = module.aoai.cognitive_account_endpoint
-  azure_openai_text_deployment_id       = var.azure_openai_text_deployment_id
-  azure_openai_text_model_name          = var.azure_openai_text_model_name
-  openai_account_id                     = module.aoai.cognitive_account_id
-  cognitive_services_endpoint           = module.cognitive_service.cognitive_account_endpoint
-  computer_vision_endpoint              = module.computer_vision.cognitive_account_endpoint
-  pdf_merge_customskill_endpoint        = "https://${module.skills.linux_function_app_default_hostname}/api/pdf_text_image_merge_skill"
-  knowledgestore_storage_account_id     = module.storage.storage_account_id
-  storage_container_name_knowledgestore = var.storage_container_name_knowledgestore
-  function_app_id                       = module.skills.skills_function_appregistration_client_id
-  public_network_access_enabled         = false
-  vnet_location                         = data.azurerm_virtual_network.virtual_network.location
-  subnet_id                             = azapi_resource.subnet_private_endpoints.id
-  private_dns_zone_id_ai_search         = var.private_dns_zone_id_ai_search
-  vision_id                             = module.computer_vision.cognitive_account_id
-  form_recognizer_id                    = module.document_intelligence.cognitive_account_id
-  cognitive_account_id                  = module.cognitive_service.cognitive_account_id
-  function_id                           = module.skills.linux_function_app_id
+  source                                        = "./modules/aisearch"
+  location                                      = var.search_service_location != "" ? var.search_service_location : var.location
+  tags                                          = local.tags
+  resource_group_name                           = azurerm_resource_group.resource_group.name
+  log_analytics_workspace_id                    = var.log_analytics_workspace_id
+  search_service_name                           = var.search_service_name != "" ? var.search_service_name : "${local.abbrs.searchServices}${local.resourceToken}"
+  search_service_sku                            = var.search_service_sku
+  semantic_search_sku                           = var.semantic_search_sku
+  search_service_partition_count                = var.search_service_partition_count
+  search_service_replica_count                  = var.search_service_replica_count
+  storage_account_id                            = module.storage.storage_account_id
+  storage_container_name_content                = var.storage_container_name_content
+  search_service_datasource_name                = var.search_service_datasource_name != "" ? var.search_service_datasource_name : "${local.abbrs.searchServices}ds-${local.resourceToken}"
+  search_service_index_name                     = var.search_service_index_name != "" ? var.search_service_index_name : "${local.abbrs.searchServices}ind-${local.resourceToken}"
+  search_service_indexer_name                   = var.search_service_indexer_name != "" ? var.search_service_indexer_name : "${local.abbrs.searchServices}inder-${local.resourceToken}"
+  search_service_skillset_name                  = var.search_service_skillset_name != "" ? var.search_service_skillset_name : "${local.abbrs.searchServices}skl-${local.resourceToken}"
+  search_service_indexer_other_file_types_name  = var.search_service_indexer_other_file_types_name != "" ? var.search_service_indexer_other_file_types_name : "${local.abbrs.searchServices}inder2-${local.resourceToken}"
+  search_service_skillset_other_file_types_name = var.search_service_skillset_other_file_types_name != "" ? var.search_service_skillset_other_file_types_name : "${local.abbrs.searchServices}skl2-${local.resourceToken}"
+  azure_openai_endpoint                         = module.aoai.cognitive_account_endpoint
+  azure_openai_text_deployment_id               = var.azure_openai_text_deployment_id
+  azure_openai_text_model_name                  = var.azure_openai_text_model_name
+  openai_account_id                             = module.aoai.cognitive_account_id
+  cognitive_services_endpoint                   = module.cognitive_service.cognitive_account_endpoint
+  computer_vision_endpoint                      = module.computer_vision.cognitive_account_endpoint
+  pdf_merge_customskill_endpoint                = "https://${module.skills.linux_function_app_default_hostname}/api/pdf_text_image_merge_skill"
+  knowledgestore_storage_account_id             = module.storage.storage_account_id
+  storage_container_name_knowledgestore         = var.storage_container_name_knowledgestore
+  function_app_id                               = module.skills.skills_function_appregistration_client_id
+  public_network_access_enabled                 = false
+  vnet_location                                 = data.azurerm_virtual_network.virtual_network.location
+  subnet_id                                     = azapi_resource.subnet_private_endpoints.id
+  private_dns_zone_id_ai_search                 = var.private_dns_zone_id_ai_search
+  vision_id                                     = module.computer_vision.cognitive_account_id
+  form_recognizer_id                            = module.document_intelligence.cognitive_account_id
+  cognitive_account_id                          = module.cognitive_service.cognitive_account_id
+  function_id                                   = module.skills.linux_function_app_id
 
   depends_on = [module.aoai, module.cognitive_service, module.storage, module.skills, module.computer_vision]
 }

deployment/terraform/infra/variables.tf

@@ -162,6 +162,17 @@ variable "search_service_skillset_name" {
   default     = ""
 }
 
+variable "search_service_indexer_other_file_types_name" {
+  description = "Specifies indexer name in the search service for other file types."
+  type        = string
+  default     = ""
+}
+
+variable "search_service_skillset_other_file_types_name" {
+  description = "Specifies skillset name in the search service for other file types."
+  type        = string
+  default     = ""
+}
 
 variable "openai_service_name" {
   description = "Specifies the sku name of the Azure OpenAI service."