feat(knowledge): add Azure AI Search backend

.secrets.baseline

@@ -306,7 +306,7 @@
         "filename": "sources/knowledge_layer/KNOWLEDGE-LAYER-SETUP.md",
         "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
         "is_verified": false,
-        "line_number": 571
+        "line_number": 577
       }
     ],
     "sources/knowledge_layer/src/foundational_rag/README.md": [

configs/config_web_azure_ai_search.yml

@@ -0,0 +1,211 @@
+# This is the Web mode configuration for Azure AI Search.
+# It has the following features:
+# - Knowledge retrieval using Azure AI Search
+# - Web search tools by default; Paper search is optional
+
+general:
+  use_uvloop: true
+  telemetry:
+    logging:
+      console:
+        _type: console
+        level: INFO
+    # tracing:
+    #   langsmith: # Optional: LangSmith tracing - requires langsmith API key. Set using `export LANGSMITH_API_KEY=<your-langsmith-api-key>`
+    #     _type: langsmith
+    #     project: nvidia-aiq
+
+  front_end:
+    _type: aiq_api
+    runner_class: aiq_api.plugin.AIQAPIWorker
+    # =========================================================================
+    # Knowledge API is automatically enabled when knowledge_retrieval function
+    # is configured
+    # =========================================================================
+    # Async Job API Settings
+    # =========================================================================
+    # Async job infrastructure database (NAT JobStore + EventStore)
+    # Used by: /v1/jobs/async routes, SSE streaming, job status persistence
+    # Requires async driver for SQLite (aiosqlite) or PostgreSQL (asyncpg)
+    # Environment overrides:
+    # - NAT_JOB_STORE_DB_URL (direct override)
+    # - NAT_JOB_STORE_DB_URL_DEV / NAT_JOB_STORE_DB_URL_PROD (via NAT_ENV)
+    db_url: ${NAT_JOB_STORE_DB_URL:-sqlite+aiosqlite:///./jobs.db}
+    # Job expiry - how long completed jobs stay in database before cleanup
+    expiry_seconds: 86400  # 24 hours (min: 600, max: 604800/7 days)
+    cors:
+      allow_origin_regex: 'http://localhost(:\d+)?|http://127.0.0.1(:\d+)?'
+      allow_methods:
+        - GET
+        - POST
+        - DELETE
+        - OPTIONS
+      allow_headers:
+        - "*"
+      allow_credentials: true
+      expose_headers:
+        - "*"
+
+llms:
+  nemotron_llm_intent:
+    _type: nim
+    model_name: nvidia/nemotron-3-super-120b-a12b
+    base_url: "https://integrate.api.nvidia.com/v1"
+    temperature: 0.5
+    top_p: 0.9
+    max_tokens: 4096
+    num_retries: 5
+    chat_template_kwargs:
+      enable_thinking: true
+
+  nemotron_super_llm:
+    _type: nim
+    model_name: nvidia/nemotron-3-super-120b-a12b
+    base_url: "https://integrate.api.nvidia.com/v1"
+    temperature: 0.7
+    top_p: 0.7
+    max_tokens: 65536
+    num_retries: 5
+    chat_template_kwargs:
+      enable_thinking: true
+
+  # LLM for document summaries (required when generate_summary: true)
+  summary_llm:
+    _type: nim
+    model_name: nvidia/nemotron-mini-4b-instruct
+    base_url: "https://integrate.api.nvidia.com/v1"
+    api_key: ${NVIDIA_API_KEY}
+    temperature: 0.3
+    max_tokens: 100
+
+functions:
+  # =========================================================================
+  # Data Source Registry
+  # =========================================================================
+  # Central registry that controls:
+  #   1. UI toggles — each source appears as an on/off switch in the frontend
+  #   2. Per-message filtering — users can select active sources per request
+  #   3. Tool auto-inheritance — agents with no explicit `tools` list receive
+  #      every tool listed here (use `exclude_tools` on agents to specialize)
+  #
+  # Adding a new tool or MCP function group? Just add it here — all agents
+  # pick it up automatically. No per-agent config changes needed.
+  #
+  # Source entry fields:
+  #   id            — Unique key used in API payloads and filtering
+  #   name          — Display name shown in the UI
+  #   description   — Human-readable description shown in the UI
+  #   tools         — List of NAT function names or function group names
+  #   requires_auth — (default: false) If true, the UI greys out this source
+  #                   until the user signs in. Use for sources that need
+  #                   user-level OAuth tokens (e.g., enterprise SSO).
+  #                   Sources using backend API keys (Tavily, Serper) should
+  #                   leave this false.
+  #   default_enabled — (default: true) Whether enabled by default
+  #
+  # See docs/source/customization/tools-and-sources.md for full details.
+  # =========================================================================
+  data_sources:
+    _type: data_source_registry
+    sources:
+      - id: web_search
+        name: "Web Search"
+        description: "Search the web for real-time information."
+        tools:
+          - web_search_tool
+          - advanced_web_search_tool
+      - id: knowledge_layer
+        name: "Knowledge Base"
+        description: "Search uploaded documents and files."
+        tools:
+          - knowledge_search
+      # Uncomment when paper_search_tool is enabled (requires SERPER_API_KEY)
+      # - id: paper_search
+      #   name: "Academic Papers"
+      #   description: "Search academic papers and scientific publications."
+      #   tools:
+      #     - paper_search_tool
+
+  web_search_tool:
+    _type: tavily_web_search
+    max_results: 5
+    max_content_length: 1000
+
+  advanced_web_search_tool:
+    _type: tavily_web_search
+    max_results: 2
+    advanced_search: true
+
+  # Knowledge Retrieval (see sources/knowledge_layer/KNOWLEDGE-LAYER-SETUP.md)
+  knowledge_search:
+    _type: knowledge_retrieval
+    backend: azure_ai_search
+    collection_name: ${COLLECTION_NAME:-aiq_default}
+    generate_summary: true
+    summary_model: summary_llm                     # Required when generate_summary: true
+    summary_db: ${AIQ_SUMMARY_DB:-sqlite+aiosqlite:///./summaries.db}
+    top_k: 5
+
+  # Paper Search (optional - requires SERPER_API_KEY)
+  # Uncomment the block below and set SERPER_API_KEY to enable academic paper search.
+  # paper_search_tool:
+  #   _type: paper_search
+  #   max_results: 5
+  #   serper_api_key: ${SERPER_API_KEY}
+
+  # =========================================================================
+  # Agents
+  # =========================================================================
+  # Tool inheritance: agents with no `tools` list inherit ALL tools from the
+  # data_source_registry above. Use `exclude_tools` to remove specific tools
+  # from an agent (e.g., give shallow the basic search, deep the advanced).
+  # To bypass auto-inherit entirely, set an explicit `tools` list.
+  # =========================================================================
+  intent_classifier:
+    _type: intent_classifier
+    llm: nemotron_llm_intent
+    # tools: omitted -> inherits all from data_source_registry
+    # exclude_tools: []
+    verbose: true
+
+  clarifier_agent:
+    _type: clarifier_agent
+    llm: nemotron_super_llm
+    planner_llm: nemotron_super_llm
+    # tools: omitted -> inherits all from data_source_registry
+    # exclude_tools: []
+    max_turns: 3
+    enable_plan_approval: true
+    log_response_max_chars: 2000
+    verbose: true
+
+  shallow_research_agent:
+    _type: shallow_research_agent
+    llm: nemotron_super_llm
+    # tools: omitted -> inherits all from data_source_registry
+    exclude_tools:                     # Remove advanced variant; shallow uses web_search_tool
+      - advanced_web_search_tool
+    verbose: true
+    max_llm_turns: 10
+    max_tool_iterations: 5
+
+  deep_research_agent:
+    _type: deep_research_agent
+    enable_citation_verification: true
+    orchestrator_llm: nemotron_super_llm
+    source_router_llm: nemotron_super_llm
+    researcher_llm: nemotron_super_llm
+    planner_llm: nemotron_super_llm
+    writer_llm: nemotron_super_llm
+    # tools: omitted -> inherits all from data_source_registry
+    exclude_tools:                     # Remove basic variant; deep uses advanced_web_search_tool
+      - web_search_tool
+    verbose: true
+
+workflow:
+  _type: chat_deepresearcher_agent
+  verbose: true
+  enable_escalation: true
+  enable_clarifier: true
+  use_async_deep_research: true
+  checkpoint_db: ${AIQ_CHECKPOINT_DB:-./checkpoints.db}

deploy/.env.example

@@ -170,6 +170,16 @@ FILE_EXPIRATION_CHECK_INTERVAL_HOURS=0
 # RAG_SERVER_URL=http://localhost:8081/v1
 # RAG_INGEST_URL=http://localhost:8082/v1
 
+# -----------------------------------------------------------------------------
+# Azure AI Search knowledge layer (required only with the Azure backend)
+# -----------------------------------------------------------------------------
+# AZURE_SEARCH_ENDPOINT=https://<service>.search.windows.net
+# API-key authentication:
+# AZURE_SEARCH_API_KEY=<search-admin-key>
+# User-assigned managed identity:
+# AZURE_CLIENT_ID=<managed-identity-client-id>
+# Optional deployment-unique prefix (default: aiq):
+# AIQ_AZURE_SEARCH_INDEX_PREFIX=aiq
 
 # -----------------------------------------------------------------------------
 # Frontend (UI) runtime configuration (Docker Compose)

docs/source/customization/configuration-reference.md

@@ -228,8 +228,8 @@ functions:
 
 ### `knowledge_retrieval`
 
-Semantic search over ingested documents. AI-Q supports three backends: LlamaIndex (local ChromaDB), Foundational RAG
-(hosted NVIDIA RAG Blueprint), and OpenSearch (self-hosted OpenSearch or Amazon OpenSearch Serverless).
+Semantic search over ingested documents. Supports LlamaIndex (local ChromaDB), Foundational RAG
+(hosted NVIDIA RAG Blueprint), OpenSearch (self-hosted OpenSearch or Amazon OpenSearch Serverless), and Azure AI Search.
 
 ```yaml
 functions:
@@ -256,9 +256,22 @@ functions:
     rag_url: ${RAG_SERVER_URL:-http://localhost:8081/v1}
     ingest_url: ${RAG_INGEST_URL:-http://localhost:8082/v1}
     timeout: 300
-    # verify_ssl: false            # Only set to false for self-signed certs
+    # verify_ssl: false # Only set to false for self-signed certs
 ```
 
+```yaml
+functions:
+  # Azure AI Search backend
+  knowledge_search:
+    _type: knowledge_retrieval
+    backend: azure_ai_search
+    collection_name: ${COLLECTION_NAME:-test_collection}
+```
+
+This example reads `AZURE_SEARCH_ENDPOINT` and `NVIDIA_API_KEY` from the
+environment. `AZURE_SEARCH_API_KEY` is optional; when absent, the adapter uses
+`DefaultAzureCredential`.
+
 ```yaml
 functions:
   # OpenSearch backend
@@ -278,7 +291,7 @@ functions:
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `backend` | `str` | `llamaindex` | Backend type: `llamaindex`, `foundational_rag`, or `opensearch`. |
+| `backend` | `str` | `llamaindex` | Backend type: `llamaindex`, `opensearch`, `foundational_rag`, or `azure_ai_search`. |
 | `collection_name` | `str` | `default` | Name of the document collection/index. |
 | `top_k` | `int` | `5` | Number of results to return per query. |
 | `generate_summary` | `bool` | `false` | Generate one-sentence summaries for ingested documents. |
@@ -289,6 +302,10 @@ functions:
 | `ingest_url` | `str` | `http://localhost:8082/v1` | RAG ingestion server URL. Foundational RAG backend only. |
 | `timeout` | `int` | `120` | Request timeout in seconds. Foundational RAG backend only. |
 | `verify_ssl` | `bool` | `true` | Verify SSL certificates. Set `false` for self-signed certs. Foundational RAG backend only. |
+| `azure_search_endpoint` | `URL` | `AZURE_SEARCH_ENDPOINT` | Azure AI Search service endpoint. Required for Azure AI Search. |
+| `azure_search_api_key` | `SecretStr` | `AZURE_SEARCH_API_KEY` | Optional admin API key. |
+| `azure_search_index_prefix` | `str` | `AIQ_AZURE_SEARCH_INDEX_PREFIX` or `aiq` | Deployment-unique namespace for the shared AI-Q index. |
+| `embed_dim` | `int` | `AIQ_EMBED_DIM` or `2048` | Embedding dimensions; must match the model and existing index schema. |
 | `opensearch_url` | `str` | `http://localhost:9200` | OpenSearch endpoint. OpenSearch backend only. |
 | `opensearch_auth_type` | `str` | `none` | Authentication mode: `none`, `basic`, or `sigv4`. |
 | `opensearch_username` | `str` | `None` | Username for basic authentication. Also read from `OPENSEARCH_USERNAME`. |
@@ -585,6 +602,7 @@ only the additional sections you need.
 |------|------|------------------------------|
 | `configs/config_cli_default.yml` | CLI | Chat pipeline with Tavily web search and clarification. No knowledge backend. Paper search is present only as a commented opt-in. |
 | `configs/config_web_default_llamaindex.yml` | Web API | Default chat pipeline with LlamaIndex/ChromaDB knowledge retrieval and Tavily. Paper search is commented out. |
+| `configs/config_web_azure_ai_search.yml` | Web API | Azure AI Search knowledge retrieval and web search |
 | `configs/config_web_frag.yml` | Web API / Helm base | Foundational RAG plus Tavily. Requires separately deployed RAG query and ingestion services. Paper search is commented out. |
 | `configs/config_web_opensearch.yml` | Web API | Built-in OpenSearch knowledge backend plus Tavily. Supports unauthenticated or basic self-hosted OpenSearch and SigV4 (`es` or `aoss`); infrastructure and credentials are deployment opt-ins. |
 | `configs/config_frontier_models.yml` | Web API | LlamaIndex plus explicit per-agent tools, Nemotron researcher roles, and an OpenAI frontier model for orchestration/planning/writing. Requires `OPENAI_API_KEY`; paper search is commented out. |

docs/source/customization/knowledge-layer.md

@@ -43,6 +43,7 @@ A pluggable abstraction for document ingestion and retrieval. Swap backends with
 |---------|-------------|------|--------------|----------|
 | `llamaindex` | `"llamaindex"` | Local Library | ChromaDB | Dev, prototyping, macOS/Linux |
 | `foundational_rag` | `"foundational_rag"` | Hosted Service | Remote Milvus | Production, multi-user |
+| `azure_ai_search` | `"azure_ai_search"` | Managed Service | Azure AI Search | Managed hybrid retrieval |
 | `opensearch` | `"opensearch"` | External Service | OpenSearch k-NN index | Self-hosted OpenSearch, Amazon OpenSearch Service, or Serverless |
 
 **Local Library Mode** - Everything runs in your Python process. No external services needed.
@@ -53,6 +54,8 @@ A pluggable abstraction for document ingestion and retrieval. Swap backends with
   - Tested with: **NVIDIA RAG Blueprint `v2.4.0`** (Helm chart `nvidia-blueprint-rag`)
   - [Deployment Guide](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md)
   - Backend-specific documentation: `sources/knowledge_layer/src/foundational_rag/README.md`
+- **`azure_ai_search`** - Stores client-generated embeddings in namespaced Azure AI Search indexes and supports
+  vector, hybrid, and semantic-ranked retrieval.
 - **`opensearch`** - Uses one vector index per AI-Q collection with `none`, `basic`, or SigV4 authentication.
   - Supports self-hosted OpenSearch, Amazon OpenSearch Service (`es`), and Amazon OpenSearch Serverless (`aoss`).
   - Can ingest in the local process or dispatch ingestion to Dask workers.
@@ -75,6 +78,7 @@ export NVIDIA_API_KEY=nvapi-your-key-here
 # 2. Install backend (choose one)
 uv pip install -e "sources/knowledge_layer[llamaindex]"        # Recommended for local dev - works on macOS/Linux
 uv pip install -e "sources/knowledge_layer[foundational_rag]"  # Requires deployed server
+uv pip install -e "sources/knowledge_layer[azure_ai_search]"   # Requires an Azure AI Search service
 uv pip install -e "sources/knowledge_layer[opensearch]"        # Requires an OpenSearch endpoint
 ```
 
@@ -164,6 +168,36 @@ functions:
     timeout: 120
 ```
 
+**Azure AI Search (Managed Service)**
+
+```yaml
+functions:
+  knowledge_search:
+    _type: knowledge_retrieval
+    backend: azure_ai_search
+    collection_name: my_docs
+```
+
+Set `AZURE_SEARCH_ENDPOINT` and `NVIDIA_API_KEY` in the environment. Setting
+`AZURE_SEARCH_API_KEY` selects key authentication; otherwise Azure
+`DefaultAzureCredential` is used. The workload identity needs `Search Service
+Contributor` for index management and `Search Index Data Contributor` for
+document ingestion and retrieval. Embedding defaults can be shared with the
+LlamaIndex backend through `AIQ_EMBED_BASE_URL` and `AIQ_EMBED_MODEL`; set
+`AIQ_EMBED_DIM` when changing the model dimensions. Set a deployment-unique
+`AIQ_AZURE_SEARCH_INDEX_PREFIX` when multiple AI-Q deployments share a search
+service.
+
+Azure stores all logical collections in one physical index selected by the
+prefix, schema version, embedding model, and dimension. Collection, file, and
+chunk manifests enforce logical isolation. Retrieval is always hybrid, and
+chunking is fixed at 1024 tokens with 128-token overlap.
+
+Upload responses return canonical UUID file IDs. Same-name uploads coexist as
+independent files. Collection cleanup uses `AIQ_COLLECTION_TTL_HOURS` (24 hours
+by default) and `AIQ_TTL_CLEANUP_INTERVAL_SECONDS` (one hour by default),
+matching the other knowledge backends.
+
 **OpenSearch (Self-Hosted or AWS)**
 
 ```yaml
@@ -264,10 +298,13 @@ File type support depends on the configured backend:
 | **LlamaIndex** | PDF, DOCX, TXT, MD, HTML, JSON, CSV |
 | **Foundational RAG** | PDF, DOCX, PPTX, TXT, MD, HTML, images (PNG, JPG) |
 | **OpenSearch** | PDF, DOCX, PPTX, TXT, MD, CSV, JSON, YAML, YML, LOG |
+| **Azure AI Search** | PDF, DOCX, TXT, MD |
 
 For custom backends, supported types are determined by the backend implementation.
 
-> **Note:** The backends support more types than the frontend currently allows. The frontend only supports uploading `.pdf,.docx,.txt,.md` (the common subset across the shipped backends). Types such as PPTX, HTML, JSON, CSV, YAML, logs, and images depend on the backend and are not accepted by the default frontend upload flow.
+> **Note:** The backends support more types than the frontend currently allows. The frontend only supports uploading
+> `.pdf,.docx,.txt,.md` (the common subset across all backends). Types like HTML, JSON, CSV, and images are supported by
+> some backends but the frontend upload flow does not handle them yet -- this is a separate task.
 
 To change the accepted types in the frontend, set `FILE_UPLOAD_ACCEPTED_TYPES` for your deployment method:
 
@@ -448,6 +485,8 @@ Configuration values are resolved in the following order (highest to lowest prio
 | `KNOWLEDGE_RETRIEVER_BACKEND` | All | Default retriever backend (fallback if not in YAML) |
 | `KNOWLEDGE_INGESTOR_BACKEND` | All | Default ingestor backend (fallback if not in YAML) |
 | `AIQ_CHROMA_DIR` | llamaindex | ChromaDB persistence path |
+| `AIQ_COLLECTION_TTL_HOURS` | all local/managed backends | Hours before stale collections are deleted (default: 24) |
+| `AIQ_TTL_CLEANUP_INTERVAL_SECONDS` | all local/managed backends | Collection cleanup interval (default: 3600) |
 | `RAG_SERVER_URL` | foundational_rag | Query server URL (port 8081) |
 | `RAG_INGEST_URL` | foundational_rag | Ingestion server URL (port 8082) |
 | `OPENSEARCH_URL` | opensearch | OpenSearch endpoint URL |