CDRIVER-4548 Support ENVIRONMENT:azure for MONGODB-OIDC. (#2166)

* add `mongoc_percent_encode`
* extend `mcd-azure.h`
  * Support a custom token resource, optional timeout, and optional client_id.
* add OIDC prose tests 5.1 and 5.2
* implement `mongoc_oidc_env_fn_azure`
* apply OIDC environment callback
  * And move check that both env+user callbacks are not set from auth to cache (to simplify).
* test more bad configuration cases
* support unified tests with Azure OIDC
* add Azure OIDC Evergreen task

Author: kevinAlbs

.evergreen/config_generator/components/oidc.py

@@ -17,7 +17,6 @@ def task_groups():
             name='test-oidc-task-group',
             tasks=['oidc-auth-test-task'],
             setup_group_can_fail_task=True,
-            setup_group_timeout_secs=60 * 60,  # 1 hour
             teardown_group_can_fail_task=True,
             teardown_group_timeout_secs=180,  # 3 minutes
             setup_group=[
@@ -35,7 +34,30 @@ def task_groups():
                     script='./drivers-evergreen-tools/.evergreen/auth_oidc/teardown.sh',
                 )
             ],
-        )
+        ),
+        EvgTaskGroup(
+            name='test-oidc-azure-task-group',
+            tasks=['oidc-azure-auth-test-task'],
+            setup_group_can_fail_task=True,
+            teardown_group_can_fail_task=True,
+            teardown_group_timeout_secs=180,  # 3 minutes
+            setup_group=[
+                FetchDET.call(),
+                ec2_assume_role(role_arn='${aws_test_secrets_role}'),
+                bash_exec(
+                    command_type=EvgCommandType.SETUP,
+                    include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'],
+                    env={'AZUREOIDC_VMNAME_PREFIX': 'CDRIVER'},
+                    script='./drivers-evergreen-tools/.evergreen/auth_oidc/azure/create-and-setup-vm.sh',
+                ),
+            ],
+            teardown_group=[
+                bash_exec(
+                    command_type=EvgCommandType.SETUP,
+                    script='./drivers-evergreen-tools/.evergreen/auth_oidc/azure/delete-vm.sh',
+                ),
+            ],
+        ),
     ]
 
 
@@ -59,7 +81,30 @@ def tasks():
                 ),
                 RunTests.call(),
             ],
-        )
+        ),
+        EvgTask(
+            name='oidc-azure-auth-test-task',
+            run_on=['debian11-small'],  # TODO: switch to 'debian11-latest' after DEVPROD-23011 fixed.
+            commands=[
+                FetchSource.call(),
+                bash_exec(
+                    working_dir='mongoc',
+                    add_expansions_to_env=True,
+                    command_type=EvgCommandType.TEST,
+                    script='.evergreen/scripts/oidc-azure-compile.sh',
+                ),
+                expansions_update(file='mongoc/oidc-remote-test-expansion.yml'),
+                bash_exec(
+                    add_expansions_to_env=True,
+                    command_type=EvgCommandType.TEST,
+                    env={
+                        'AZUREOIDC_DRIVERS_TAR_FILE': '${OIDC_TEST_TARBALL}',
+                        'AZUREOIDC_TEST_CMD': 'source ./env.sh && ./.evergreen/scripts/oidc-azure-test.sh',
+                    },
+                    script='./drivers-evergreen-tools/.evergreen/auth_oidc/azure/run-driver-test.sh',
+                ),
+            ],
+        ),
     ]
 
 
@@ -68,7 +113,6 @@ def variants():
         BuildVariant(
             name='oidc',
             display_name='OIDC',
-            run_on=[find_small_distro('ubuntu2404').name],
-            tasks=[EvgTaskRef(name='test-oidc-task-group')],
+            tasks=[EvgTaskRef(name='test-oidc-task-group'), EvgTaskRef(name='test-oidc-azure-task-group')],
         ),
     ]

.evergreen/generated_configs/task_groups.yml

@@ -1,4 +1,35 @@
 task_groups:
+  - name: test-oidc-azure-task-group
+    setup_group:
+      - func: fetch-det
+      - command: ec2.assume_role
+        params:
+          role_arn: ${aws_test_secrets_role}
+      - command: subprocess.exec
+        type: setup
+        params:
+          binary: bash
+          env:
+            AZUREOIDC_VMNAME_PREFIX: CDRIVER
+          include_expansions_in_env:
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - AWS_SESSION_TOKEN
+          args:
+            - -c
+            - ./drivers-evergreen-tools/.evergreen/auth_oidc/azure/create-and-setup-vm.sh
+    setup_group_can_fail_task: true
+    tasks:
+      - oidc-azure-auth-test-task
+    teardown_group:
+      - command: subprocess.exec
+        type: setup
+        params:
+          binary: bash
+          args:
+            - -c
+            - ./drivers-evergreen-tools/.evergreen/auth_oidc/azure/delete-vm.sh
+    teardown_group_timeout_secs: 180
   - name: test-oidc-task-group
     setup_group:
       - func: fetch-det
@@ -17,7 +48,6 @@ task_groups:
             - -c
             - ./drivers-evergreen-tools/.evergreen/auth_oidc/setup.sh
     setup_group_can_fail_task: true
-    setup_group_timeout_secs: 3600
     tasks:
       - oidc-auth-test-task
     teardown_group:

.evergreen/generated_configs/tasks.yml

@@ -4221,6 +4221,34 @@ tasks:
             - { key: MONGODB_VERSION, value: latest }
             - { key: TOPOLOGY, value: replica_set }
       - func: run-tests
+  - name: oidc-azure-auth-test-task
+    run_on:
+      - debian11-small
+    commands:
+      - func: fetch-source
+      - command: subprocess.exec
+        type: test
+        params:
+          binary: bash
+          working_dir: mongoc
+          add_expansions_to_env: true
+          args:
+            - -c
+            - .evergreen/scripts/oidc-azure-compile.sh
+      - command: expansions.update
+        params:
+          file: mongoc/oidc-remote-test-expansion.yml
+      - command: subprocess.exec
+        type: test
+        params:
+          binary: bash
+          add_expansions_to_env: true
+          env:
+            AZUREOIDC_DRIVERS_TAR_FILE: ${OIDC_TEST_TARBALL}
+            AZUREOIDC_TEST_CMD: source ./env.sh && ./.evergreen/scripts/oidc-azure-test.sh
+          args:
+            - -c
+            - ./drivers-evergreen-tools/.evergreen/auth_oidc/azure/run-driver-test.sh
   - name: openssl-compat-1.0.2-shared-ubuntu2404-gcc
     run_on: ubuntu2404-large
     tags: [openssl-compat, openssl-1.0.2, openssl-shared, ubuntu2404, gcc]

.evergreen/generated_configs/variants.yml

@@ -255,10 +255,9 @@ buildvariants:
       - name: mock-server-test
   - name: oidc
     display_name: OIDC
-    run_on:
-      - ubuntu2404-small
     tasks:
       - name: test-oidc-task-group
+      - name: test-oidc-azure-task-group
   - name: openssl-compat-matrix
     display_name: OpenSSL Compatibility Matrix
     tasks:

.evergreen/scripts/oidc-azure-compile.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+if [[ "${distro_id:?}" == "debian11-small" ]]; then
+  # Temporary workaround for lack of uv on `debian11`. TODO: remove after DEVPROD-23011 is resolved.
+  uv_dir="$(mktemp -d)"
+  python3 -m virtualenv "${uv_dir:?}"
+  # shellcheck source=/dev/null
+  (. "${uv_dir:?}/bin/activate" && python -m pip install uv)
+  PATH="${uv_dir:?}/bin:${PATH:-}"
+  command -V uv >/dev/null
+fi
+
+. .evergreen/scripts/install-build-tools.sh
+install_build_tools
+export CMAKE_GENERATOR="Ninja"
+
+# Use ccache if able.
+. .evergreen/scripts/find-ccache.sh
+find_ccache_and_export_vars "$(pwd)" || true
+
+echo "Compile test-libmongoc ... begin"
+# Disable unnecessary dependencies. test-libmongoc is copied to a remote host for testing, which may not have all dependent libraries.
+cmake_flags=(
+  -DENABLE_SASL=OFF
+  -DENABLE_SNAPPY=OFF
+  -DENABLE_ZSTD=OFF
+  -DENABLE_ZLIB=OFF
+  -DENABLE_SRV=OFF
+  -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF
+  -DENABLE_EXAMPLES=OFF
+  -DENABLE_SRV=OFF
+)
+cmake "${cmake_flags[@]}" -Bcmake-build
+cmake --build cmake-build --target test-libmongoc
+echo "Compile test-libmongoc ... end"
+
+# Create tarball for remote testing.
+echo "Creating test-libmongoc tarball ... begin"
+
+# Copy test binary and JSON test files. All JSON test files are needed to start test-libmongoc.
+files=(
+  .evergreen/scripts/oidc-azure-test.sh
+  cmake-build/src/libmongoc/test-libmongoc
+  src/libmongoc/tests/json
+  src/libbson/tests/json
+)
+tar -czf test-libmongoc.tar.gz "${files[@]}"
+echo "Creating test-libmongoc tarball ... end"
+
+cat <<EOT >oidc-remote-test-expansion.yml
+OIDC_TEST_TARBALL: $(pwd)/test-libmongoc.tar.gz
+EOT

.evergreen/scripts/oidc-azure-test.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+export MONGOC_TEST_OIDC="ON"
+export MONGOC_TEST_USER="$OIDC_ADMIN_USER"
+export MONGOC_TEST_PASSWORD="$OIDC_ADMIN_PWD"
+export MONGOC_AZURE_RESOURCE="$AZUREOIDC_RESOURCE"
+
+# Install required OpenSSL runtime library.
+sudo apt install -y libssl-dev
+
+./cmake-build/src/libmongoc/test-libmongoc -d -l '/auth/unified/*' -l '/oidc/*'

CONTRIBUTING.md

@@ -321,6 +321,7 @@ To run test cases with large allocations, set:
 * `MONGOC_TEST_LARGE_ALLOCATIONS=on` This may result in sudden test suite termination due to allocation failure. Use with caution.
 
 * `MONGOC_TEST_OIDC=on` to test OIDC using a test environment described [here](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/d7a7337b384392a09fbe7fc80a7244e6f1226c18/.evergreen/auth_oidc).
+* `MONGOC_AZURE_RESOURCE=<resource>` to test OIDC using an Azure test environment described [here](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/d7a7337b384392a09fbe7fc80a7244e6f1226c18/.evergreen/auth_oidc/azure/README.md).
 
 All tests should pass before submitting a patch.
 

src/libmongoc/src/mongoc/mcd-azure.c

@@ -21,21 +21,28 @@
 
 #include <mlib/cmp.h>
 #include <mlib/duration.h>
+#include <mlib/time_point.h>
 #include <mlib/timer.h>
 
 #define AZURE_API_VERSION "2018-02-01"
 
-static const char *const DEFAULT_METADATA_PATH =
-   "/metadata/identity/oauth2/"
-   "token?api-version=" AZURE_API_VERSION "&resource=https%3A%2F%2Fvault.azure.net";
+static const char *const DEFAULT_METADATA_PATH = "/metadata/identity/oauth2/token?api-version=" AZURE_API_VERSION;
 
-void
+bool
 mcd_azure_imds_request_init(mcd_azure_imds_request *req,
+                            const char *token_resource,
                             const char *const opt_imds_host,
                             int opt_port,
-                            const char *const opt_extra_headers)
+                            const char *const opt_extra_headers,
+                            const char *const opt_client_id)
 {
    BSON_ASSERT_PARAM(req);
+   BSON_ASSERT_PARAM(token_resource);
+
+   bool ok = false;
+   char *encoded_token_resource = NULL;
+   mcommon_string_append_t path = {0};
+
    _mongoc_http_request_init(&req->req);
    // The HTTP host of the IMDS server
    req->req.host = req->_owned_host = bson_strdup(opt_imds_host ? opt_imds_host : "169.254.169.254");
@@ -52,9 +59,33 @@ mcd_azure_imds_request_init(mcd_azure_imds_request *req,
    req->req.extra_headers = req->_owned_headers = bson_strdup_printf("Metadata: true\r\n"
                                                                      "Accept: application/json\r\n%s",
                                                                      opt_extra_headers ? opt_extra_headers : "");
-   // The default path is suitable. In the future, we may want to add query
-   // parameters to disambiguate a managed identity.
-   req->req.path = req->_owned_path = bson_strdup(DEFAULT_METADATA_PATH);
+   // Build the path with query parameters.
+   encoded_token_resource = mongoc_percent_encode(token_resource);
+   if (!encoded_token_resource) {
+      goto fail;
+   }
+
+   mcommon_string_new_as_append(&path);
+
+   if (!mcommon_string_append(&path, DEFAULT_METADATA_PATH) ||
+       !mcommon_string_append_printf(&path, "&resource=%s", encoded_token_resource)) {
+      goto fail;
+   }
+
+   if (opt_client_id) {
+      if (!mcommon_string_append_printf(&path, "&client_id=%s", opt_client_id)) {
+         goto fail;
+      }
+   }
+
+   req->req.path = req->_owned_path = mcommon_string_from_append_destroy_with_steal(&path);
+   path = (mcommon_string_append_t){0};
+
+   ok = true;
+fail:
+   bson_free(encoded_token_resource);
+   mcommon_string_from_append_destroy(&path);
+   return ok;
 }
 
 void
@@ -158,11 +189,15 @@ mcd_azure_access_token_destroy(mcd_azure_access_token *c)
 
 bool
 mcd_azure_access_token_from_imds(mcd_azure_access_token *const out,
+                                 const char *token_resource,
                                  const char *const opt_imds_host,
                                  int opt_port,
                                  const char *opt_extra_headers,
+                                 mlib_timer opt_timer,
+                                 const char *opt_client_id,
                                  bson_error_t *error)
 {
+   BSON_ASSERT_PARAM(token_resource);
    BSON_ASSERT_PARAM(out);
 
    bool okay = false;
@@ -174,9 +209,16 @@ mcd_azure_access_token_from_imds(mcd_azure_access_token *const out,
    _mongoc_http_response_init(&resp);
 
    mcd_azure_imds_request req = MCD_AZURE_IMDS_REQUEST_INIT;
-   mcd_azure_imds_request_init(&req, opt_imds_host, opt_port, opt_extra_headers);
+   if (!mcd_azure_imds_request_init(&req, token_resource, opt_imds_host, opt_port, opt_extra_headers, opt_client_id)) {
+      _mongoc_set_error(error, MONGOC_ERROR_AZURE, MONGOC_ERROR_KMS_SERVER_HTTP, "Failed to initialize request");
+      goto fail;
+   }
+
+   mlib_timer timer = mlib_time_cmp(opt_timer.expires_at, ==, (mlib_time_point){0})
+                         ? opt_timer
+                         : mlib_expires_after(mlib_duration(3, s)); // Default 3 second timeout.
 
-   if (!_mongoc_http_send(&req.req, mlib_expires_after(mlib_duration(3, s)), false, NULL, &resp, error)) {
+   if (!_mongoc_http_send(&req.req, timer, false, NULL, &resp, error)) {
       goto fail;
    }