diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f0bbf15013..737ee4570c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -1312,11 +1312,11 @@ jobs:
- provider: authentik
runtime: authentik-compose
backend: compose
- command: compose
+ command: test compose
- provider: authentik
runtime: authentik-kubernetes
backend: kubernetes
- command: k8s
+ command: test k8s
env:
NMP_AUTHENTIK_K8S_RUNTIME: kind
NMP_AUTHENTIK_K8S_NAMESPACE: nemo-authentik
diff --git a/contrib/auth/authentik/README.md b/contrib/auth/authentik/README.md
index a9f73a3ad3..de396f71c4 100644
--- a/contrib/auth/authentik/README.md
+++ b/contrib/auth/authentik/README.md
@@ -32,9 +32,9 @@ Runtime details live separately:
## Test Harness
`run.sh` is the automation entrypoint for CI-style validation and repeatable
-local test runs. It can run the local Compose stack, run the Compose auth-idp
-contract tests, run the Kubernetes auth-idp contract tests, and clean up local
-resources.
+local test runs. It can start durable Compose or Kubernetes auth-idp
+environments, run the Compose or Kubernetes auth-idp contract tests, and clean
+up local resources.
```bash
contrib/auth/authentik/run.sh --help
@@ -43,17 +43,46 @@ contrib/auth/authentik/run.sh --help
Common commands:
```bash
-contrib/auth/authentik/run.sh compose
-contrib/auth/authentik/run.sh k8s
+contrib/auth/authentik/run.sh up compose
+contrib/auth/authentik/run.sh up k8s
+contrib/auth/authentik/run.sh test compose
+contrib/auth/authentik/run.sh test k8s
contrib/auth/authentik/run.sh prepare-local
-contrib/auth/authentik/run.sh run-local
-contrib/auth/authentik/run.sh down
+contrib/auth/authentik/run.sh down compose
+contrib/auth/authentik/run.sh down k8s
+contrib/auth/authentik/run.sh clean
```
+`up compose` and `up k8s` also create user NeMo CLI contexts named
+`authentik-compose` and `authentik-k8s`. The contexts include the
+local gateway URL, default workspace, and gateway certificate authority so users
+can switch to them with `nemo config use-context`.
+
+Use `--key KEY` with `up` to run a second durable instance. The key derives
+managed names such as `authentik-compose-KEY`, `authentik-k8s-KEY`,
+`authentik-e2e-KEY`, and `nmp-authentik-KEY`, and keyed `up` commands choose
+an available local gateway port by default. Use the same key with `down` to
+remove that instance:
+
+```bash
+contrib/auth/authentik/run.sh up compose --key dev
+contrib/auth/authentik/run.sh down compose --key dev
+```
+
+`up k8s` uses `https://127.0.0.1:18082` by default for a stable manual URL.
+`test k8s` chooses an available local port by default so it can run while other
+local k8s auth-idp workflows are using that stable port. Set
+`NMP_AUTHENTIK_K8S_GATEWAY_PORT` to force a specific test port. The `test`
+actions do not add user NeMo CLI contexts.
+
The harness keeps generated local inputs in `contrib/auth/authentik/.generated`
so Compose and Kubernetes test runs can reuse the same workload-token signing
-key. Diagnostics are written under `docker/logs/authentik-*` by default, or
-under `E2E_SERVICES_LOG_DIR` when that environment variable is set.
+key. Durable `up` actions record lifecycle state under
+`contrib/auth/authentik/.generated/instances` by default so `down` and `clean`
+can remove recorded instances later. Override that path with
+`NEMO_AUTHENTIK_STATE_DIR`. Diagnostics are written under
+`docker/logs/authentik-*` by default, or under `E2E_SERVICES_LOG_DIR` when that
+environment variable is set.
For manual startup and walkthroughs, prefer the shared tutorial above.
diff --git a/contrib/auth/authentik/helm/values.yaml b/contrib/auth/authentik/helm/values.yaml
index d82be64cbd..9b3fafff19 100644
--- a/contrib/auth/authentik/helm/values.yaml
+++ b/contrib/auth/authentik/helm/values.yaml
@@ -130,6 +130,8 @@ nemo-platform:
core:
controller:
controllerGroup: core
+ startupProbe:
+ failureThreshold: 80
env:
NMP_PLATFORM_URL: "https://nemo-platform-envoy.$(POD_NAMESPACE).svc.cluster.local:8080"
NMP_AUTH_URL: "https://nemo-platform-envoy.$(POD_NAMESPACE).svc.cluster.local:8080"
@@ -141,6 +143,8 @@ nemo-platform:
create: true
name: nemo
serviceGroup: core
+ startupProbe:
+ failureThreshold: 80
env:
NMP_AUTH_TOKEN_SIGNING__PRIVATE_KEY_FILE: "/etc/nmp/workload-token/private-key.pem"
extraVolumes:
diff --git a/contrib/auth/authentik/run.sh b/contrib/auth/authentik/run.sh
index 8d0a181220..6e21865d6d 100755
--- a/contrib/auth/authentik/run.sh
+++ b/contrib/auth/authentik/run.sh
@@ -9,8 +9,10 @@ AUTHENTIK_ROOT="${SCRIPT_DIR}"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)"
ACTION=""
+TARGET=""
COMPOSE_DIR="${AUTHENTIK_ROOT}/compose"
DRY_RUN="false"
+INSTANCE_KEY=""
IMAGE_SELECTED="false"
IMAGE_REGISTRY="${IMAGE_REGISTRY:-my-registry}"
BAKE_TAG="${BAKE_TAG:-local}"
@@ -20,15 +22,17 @@ TEST_PLATFORM=""
TEST_PLATFORM_SET="false"
TEST_DOCKER_TARGET="nmp-api-docker"
COMPOSE_DIR_SET="false"
-REUSE_COMPOSE_PROJECT_NAME="${NMP_AUTHENTIK_COMPOSE_PROJECT_NAME:-authentik-e2e-reuse}"
-REUSE_COMPOSE_GATEWAY_PORT="${NMP_AUTHENTIK_COMPOSE_GATEWAY_PORT:-18083}"
-REUSE_COMPOSE_GATEWAY_TLS_VOLUME="${NMP_AUTHENTIK_COMPOSE_GATEWAY_TLS_VOLUME:-authentik-e2e-${REUSE_COMPOSE_GATEWAY_PORT}-gateway-tls}"
-REUSE_COMPOSE_WORKLOAD_NETWORK_NAME="${NMP_AUTHENTIK_COMPOSE_WORKLOAD_NETWORK_NAME:-authentik-e2e-${REUSE_COMPOSE_GATEWAY_PORT}-workload}"
-REUSE_K8S_CLUSTER_NAME="${NMP_AUTHENTIK_K8S_REUSE_CLUSTER_NAME:-nmp-authentik-reuse}"
-K8S_GATEWAY_PORT="${NMP_AUTHENTIK_K8S_GATEWAY_PORT:-18082}"
+REUSE_COMPOSE_PROJECT_NAME="${NMP_AUTHENTIK_COMPOSE_PROJECT_NAME:-}"
+REUSE_COMPOSE_GATEWAY_PORT="${NMP_AUTHENTIK_COMPOSE_GATEWAY_PORT:-}"
+REUSE_COMPOSE_GATEWAY_TLS_VOLUME="${NMP_AUTHENTIK_COMPOSE_GATEWAY_TLS_VOLUME:-}"
+REUSE_COMPOSE_WORKLOAD_NETWORK_NAME="${NMP_AUTHENTIK_COMPOSE_WORKLOAD_NETWORK_NAME:-}"
+REUSE_K8S_CLUSTER_NAME="${NMP_AUTHENTIK_K8S_REUSE_CLUSTER_NAME:-}"
+DEFAULT_K8S_GATEWAY_PORT="18082"
+K8S_GATEWAY_PORT="${NMP_AUTHENTIK_K8S_GATEWAY_PORT:-}"
K8S_JUNIT_XML="${NMP_AUTHENTIK_K8S_JUNIT_XML:-report-auth-idp-kubernetes.xml}"
HELM_NAMESPACE="${HELM_NAMESPACE:-${NMP_AUTHENTIK_K8S_NAMESPACE:-nemo-authentik}}"
HELM_RELEASE="${HELM_RELEASE:-${NMP_AUTHENTIK_K8S_HELM_RELEASE:-authentik-demo}}"
+HELM_WAIT_TIMEOUT="${HELM_WAIT_TIMEOUT:-${NMP_AUTHENTIK_K8S_HELM_WAIT_TIMEOUT:-20m}}"
K8S_CLUSTER_NAME="${NMP_AUTHENTIK_K8S_CLUSTER_NAME:-}"
K8S_RUNTIME="${NMP_AUTHENTIK_K8S_RUNTIME:-kind}"
K8S_RUNTIME_SET="false"
@@ -38,6 +42,7 @@ K8S_SKIP_IMAGE_LOAD="${NMP_AUTHENTIK_K8S_SKIP_IMAGE_LOAD:-0}"
K8S_SKIP_IMAGE_LOAD_SET="false"
K8S_NGC_EXISTING_SECRET="${NMP_AUTHENTIK_K8S_NGC_EXISTING_SECRET:-}"
K8S_IMAGE_PULL_SECRET="${NMP_AUTHENTIK_K8S_IMAGE_PULL_SECRET:-}"
+AUTHENTIK_WORKSPACE="${NMP_AUTHENTIK_WORKSPACE:-authentik-demo}"
diagnostics_dir() {
local mode="$1"
@@ -89,26 +94,75 @@ write_diagnostics_metadata() {
} >"${output}/run-metadata.txt"
}
+choose_free_tcp_port() {
+ python3 -c 'import socket
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+ sock.bind(("127.0.0.1", 0))
+ print(sock.getsockname()[1])'
+}
+
+validate_k8s_gateway_port() {
+ if [[ ! "${K8S_GATEWAY_PORT}" =~ ^[0-9]+$ ]]; then
+ die "NMP_AUTHENTIK_K8S_GATEWAY_PORT must be an integer TCP port"
+ fi
+ if ((K8S_GATEWAY_PORT < 1 || K8S_GATEWAY_PORT > 65535)); then
+ die "NMP_AUTHENTIK_K8S_GATEWAY_PORT must be between 1 and 65535"
+ fi
+}
+
+configure_k8s_gateway_port() {
+ if [[ "${TARGET}" != "k8s" ]]; then
+ return
+ fi
+ if [[ "${ACTION}" != "up" && "${ACTION}" != "test" ]]; then
+ return
+ fi
+ if [[ -z "${K8S_GATEWAY_PORT}" ]]; then
+ case "${ACTION}" in
+ up)
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ K8S_GATEWAY_PORT="$(choose_free_tcp_port)"
+ else
+ K8S_GATEWAY_PORT="${DEFAULT_K8S_GATEWAY_PORT}"
+ fi
+ ;;
+ test)
+ K8S_GATEWAY_PORT="$(choose_free_tcp_port)"
+ ;;
+ esac
+ fi
+ validate_k8s_gateway_port
+}
+
usage() {
cat <<'EOF'
Usage:
- contrib/auth/authentik/run.sh run-local [options]
- contrib/auth/authentik/run.sh down [options]
+ contrib/auth/authentik/run.sh up compose [options]
+ contrib/auth/authentik/run.sh up k8s [options]
+ contrib/auth/authentik/run.sh test compose [options]
+ contrib/auth/authentik/run.sh test k8s [options]
+ contrib/auth/authentik/run.sh down [compose|k8s|all] [options]
+ contrib/auth/authentik/run.sh clean [compose|k8s|all] [options]
contrib/auth/authentik/run.sh prepare-local [options]
- contrib/auth/authentik/run.sh compose [options]
contrib/auth/authentik/run.sh render-blueprint [options]
- contrib/auth/authentik/run.sh k8s [options]
Runs the local Authentik reference example or its auth-idp test suite.
Actions:
- run-local Start NeMo, Authentik, and the gateway in the foreground.
- down Remove the example Compose stack, reused test Compose stack,
- and reused Kubernetes test cluster.
+ up compose Start the reusable Compose auth-idp stack and skip tests.
+ up k8s Create or reuse the Kubernetes auth-idp cluster, install
+ the Helm chart, start a local gateway port-forward, and
+ skip tests.
+ test compose Build the local test image if needed, then run Compose
+ auth-idp tests.
+ test k8s Run the Helm-based kind/k3d Kubernetes E2E test.
+ down compose Remove managed Authentik Compose instances and contexts.
+ down k8s Remove managed Authentik Kubernetes instances and contexts.
+ down all Remove managed Compose and Kubernetes instances.
+ Bare "down" defaults to "down all".
+ clean Remove stale recorded instances using down semantics.
prepare-local Create generated local inputs without starting Compose.
- compose Build the local test image if needed, then run Compose auth-idp tests.
render-blueprint Copy the checked-in Authentik blueprint into generated inputs.
- k8s Run the Helm-based kind/k3d Kubernetes E2E test.
Image options:
--image IMAGE Use an existing nmp-api image.
@@ -120,10 +174,12 @@ Test options:
on gateway port 18083.
For k8s, use cluster nmp-authentik-reuse with
the selected runtime, creating it if needed and
- keeping it after the run.
- The k8s runner uses gateway port 18082 by default
- to avoid the tutorial's 18081 port. Override with
- NMP_AUTHENTIK_K8S_GATEWAY_PORT.
+ keeping it after the run. The up actions use these
+ reusable resources by default.
+ The k8s up action uses gateway port 18082 by default
+ to avoid the tutorial's 18081 port. The k8s test
+ action chooses a free local port by default. Override
+ either with NMP_AUTHENTIK_K8S_GATEWAY_PORT.
--platform PLATFORM Platform for the default local test image build.
Applies to compose and k8s.
Default: current machine architecture.
@@ -133,23 +189,31 @@ Test options:
For a fresh cluster, use only with an explicit pullable
--image.
+Instance options:
+ --key KEY Manage a named up/down instance. The default instance
+ uses contexts authentik-compose and authentik-k8s.
+ A key derives names such as authentik-compose-KEY,
+ authentik-k8s-KEY, and matching local resources.
+
Other options:
- --compose-dir DIR Compose directory for run-local/down. Default: contrib/auth/authentik/compose.
+ --compose-dir DIR Compose directory for down/compose.
+ Default: contrib/auth/authentik/compose.
--dry-run Print commands without running them.
-h, --help Show this help.
Examples:
- contrib/auth/authentik/run.sh run-local
- contrib/auth/authentik/run.sh run-local --image my-registry/nmp-api:local
+ contrib/auth/authentik/run.sh up compose
+ contrib/auth/authentik/run.sh up compose --key dev
+ contrib/auth/authentik/run.sh up k8s
+ contrib/auth/authentik/run.sh up k8s --key dev
+ contrib/auth/authentik/run.sh test compose
+ contrib/auth/authentik/run.sh test k8s
+ contrib/auth/authentik/run.sh down compose
+ contrib/auth/authentik/run.sh down compose --key dev
+ contrib/auth/authentik/run.sh down k8s
+ contrib/auth/authentik/run.sh clean
contrib/auth/authentik/run.sh prepare-local
- contrib/auth/authentik/run.sh compose
- contrib/auth/authentik/run.sh compose --reuse
- contrib/auth/authentik/run.sh compose --image my-registry/nmp-api:local
contrib/auth/authentik/run.sh render-blueprint
- contrib/auth/authentik/run.sh k8s
- contrib/auth/authentik/run.sh k8s --runtime k3d
- contrib/auth/authentik/run.sh k8s --reuse
- contrib/auth/authentik/run.sh k8s --reuse --skip-image-load
contrib/auth/authentik/run.sh down
EOF
}
@@ -181,6 +245,91 @@ parse_image() {
fi
}
+validate_instance_key() {
+ if [[ -z "${INSTANCE_KEY}" ]]; then
+ return
+ fi
+ if [[ ! "${INSTANCE_KEY}" =~ ^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$ ]]; then
+ die "--key must use 1-32 lowercase letters, digits, and hyphens, and start/end with a letter or digit"
+ fi
+}
+
+instance_suffix() {
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ printf -- "-%s" "${INSTANCE_KEY}"
+ fi
+}
+
+compose_context_name() {
+ printf "authentik-compose%s" "$(instance_suffix)"
+}
+
+k8s_context_name() {
+ printf "authentik-k8s%s" "$(instance_suffix)"
+}
+
+key_option_suffix() {
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ printf " --key %s" "${INSTANCE_KEY}"
+ fi
+}
+
+state_matches_instance_key() {
+ local state_file="$1"
+ local state_key
+
+ if [[ -z "${INSTANCE_KEY}" ]]; then
+ return 0
+ fi
+
+ state_key="$(read_lifecycle_field "${state_file}" instance_key)"
+ [[ "${state_key}" == "${INSTANCE_KEY}" ]]
+}
+
+configure_instance_defaults() {
+ validate_instance_key
+
+ if [[ -z "${REUSE_COMPOSE_PROJECT_NAME}" ]]; then
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ REUSE_COMPOSE_PROJECT_NAME="authentik-e2e-${INSTANCE_KEY}"
+ else
+ REUSE_COMPOSE_PROJECT_NAME="authentik-e2e-reuse"
+ fi
+ fi
+
+ if [[ -z "${REUSE_COMPOSE_GATEWAY_PORT}" ]]; then
+ if [[ "${ACTION}" == "up" && "${TARGET}" == "compose" && -n "${INSTANCE_KEY}" ]]; then
+ REUSE_COMPOSE_GATEWAY_PORT="$(choose_free_tcp_port)"
+ else
+ REUSE_COMPOSE_GATEWAY_PORT="18083"
+ fi
+ fi
+
+ if [[ -z "${REUSE_COMPOSE_GATEWAY_TLS_VOLUME}" ]]; then
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ REUSE_COMPOSE_GATEWAY_TLS_VOLUME="authentik-e2e-${INSTANCE_KEY}-gateway-tls"
+ else
+ REUSE_COMPOSE_GATEWAY_TLS_VOLUME="authentik-e2e-${REUSE_COMPOSE_GATEWAY_PORT}-gateway-tls"
+ fi
+ fi
+
+ if [[ -z "${REUSE_COMPOSE_WORKLOAD_NETWORK_NAME}" ]]; then
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ REUSE_COMPOSE_WORKLOAD_NETWORK_NAME="authentik-e2e-${INSTANCE_KEY}-workload"
+ else
+ REUSE_COMPOSE_WORKLOAD_NETWORK_NAME="authentik-e2e-${REUSE_COMPOSE_GATEWAY_PORT}-workload"
+ fi
+ fi
+
+ if [[ -z "${REUSE_K8S_CLUSTER_NAME}" ]]; then
+ if [[ -n "${INSTANCE_KEY}" ]]; then
+ REUSE_K8S_CLUSTER_NAME="nmp-authentik-${INSTANCE_KEY}"
+ else
+ REUSE_K8S_CLUSTER_NAME="nmp-authentik-reuse"
+ fi
+ fi
+}
+
host_platform() {
case "$(uname -m)" in
x86_64 | amd64)
@@ -268,6 +417,87 @@ gateway_tls_key_file() {
printf "%s/tls.key" "$(gateway_tls_dir)"
}
+lifecycle_state_dir() {
+ local configured="${NEMO_AUTHENTIK_STATE_DIR:-./.generated/instances}"
+ if [[ "${configured}" = /* ]]; then
+ printf "%s" "${configured}"
+ else
+ printf "%s/%s" "${AUTHENTIK_ROOT}" "${configured#./}"
+ fi
+}
+
+safe_state_id() {
+ printf "%s" "$1" | tr -c '[:alnum:]_.-' '_'
+}
+
+lifecycle_state_file() {
+ local target="$1"
+ local instance_id="$2"
+
+ printf "%s/%s-%s.json" "$(lifecycle_state_dir)" "${target}" "$(safe_state_id "${instance_id}")"
+}
+
+write_lifecycle_state() {
+ local target="$1"
+ local instance_id="$2"
+ local state_file
+ shift 2
+
+ state_file="$(lifecycle_state_file "${target}" "${instance_id}")"
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ printf "+ mkdir -p %q\n" "$(dirname -- "${state_file}")"
+ printf "+ write lifecycle state %q\n" "${state_file}"
+ return
+ fi
+
+ mkdir -p "$(dirname -- "${state_file}")"
+ python3 - "${state_file}" "$@" <<'PY'
+import json
+import sys
+from datetime import datetime, timezone
+
+path = sys.argv[1]
+data = {
+ "version": 1,
+ "created_at_utc": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+}
+for item in sys.argv[2:]:
+ key, value = item.split("=", 1)
+ data[key] = value
+
+with open(path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=2, sort_keys=True)
+ f.write("\n")
+PY
+}
+
+read_lifecycle_field() {
+ local state_file="$1"
+ local field="$2"
+
+ python3 - "${state_file}" "${field}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as f:
+ value = json.load(f).get(sys.argv[2], "")
+if value is None:
+ value = ""
+print(value)
+PY
+}
+
+remove_lifecycle_state_file() {
+ local state_file="$1"
+
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ printf "+ rm -f %q\n" "${state_file}"
+ return
+ fi
+
+ rm -f "${state_file}"
+}
+
ensure_gateway_tls_certificate() {
local output_dir
local cert
@@ -438,33 +668,611 @@ run_in_repo() {
(cd "${REPO_ROOT}" && "$@")
}
+run_command() {
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ print_command "$@"
+ return
+ fi
+
+ "$@"
+}
+
+register_nemo_context() {
+ local context_name="$1"
+ local gateway_url="$2"
+ local ca_bundle="$3"
+
+ echo "Registering NeMo context ${context_name}: ${gateway_url}"
+ run_in_repo uv run --frozen nemo config set \
+ --context "${context_name}" \
+ --base-url "${gateway_url}" \
+ --certificate-authority "${ca_bundle}" \
+ --workspace "${AUTHENTIK_WORKSPACE}"
+}
+
+delete_nemo_context() {
+ local context_name="$1"
+ local status
+
+ if [[ -z "${context_name}" ]]; then
+ return
+ fi
+
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ run_in_repo uv run --frozen nemo config delete-context "${context_name}" --prune-orphans
+ return
+ fi
+
+ set +e
+ (cd "${REPO_ROOT}" && uv run --frozen nemo config delete-context "${context_name}" --prune-orphans)
+ status="$?"
+ set -e
+ if [[ "${status}" -ne 0 ]]; then
+ echo "Warning: failed to delete NeMo context ${context_name}" >&2
+ fi
+}
+
+wait_for_https_ready() {
+ local url="$1"
+ local ca_cert="$2"
+ local timeout_seconds="${3:-180}"
+ local started="${SECONDS}"
+
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ print_command curl -fsS --cacert "${ca_cert}" "${url}"
+ return
+ fi
+
+ if ! command -v curl >/dev/null 2>&1; then
+ die "curl is required to wait for ${url}"
+ fi
+
+ until curl -fsS --cacert "${ca_cert}" "${url}" >/dev/null 2>&1; do
+ if ((SECONDS - started >= timeout_seconds)); then
+ echo "timed out waiting for ${url}" >&2
+ return 1
+ fi
+ sleep 2
+ done
+}
+
prepare_local() {
render_blueprint
ensure_workload_token_private_key
ensure_gateway_tls_certificate
}
-run_local() {
- echo "Using existing NeMo API image: $(image_ref)"
- prepare_local
+compose_up() {
+ local gateway_url="https://127.0.0.1:${REUSE_COMPOSE_GATEWAY_PORT}"
+ local ca_bundle
+ local context_name
- if [[ "${DRY_RUN}" == "true" ]]; then
- run_with_compose_env_in_dir "${COMPOSE_DIR}" docker compose up
- return
+ TEST_LIFECYCLE="reuse"
+ prepare_local
+ ca_bundle="$(gateway_tls_cert_file)"
+ context_name="$(compose_context_name)"
+ if [[ "${IMAGE_SELECTED}" == "true" ]]; then
+ echo "Using prebuilt auth-idp Compose image: $(image_ref)"
+ else
+ build_default_test_image
fi
- trap 'run_with_compose_env_in_dir "${COMPOSE_DIR}" docker compose down -v' EXIT INT TERM
- run_with_compose_env_in_dir "${COMPOSE_DIR}" docker compose up
+ run_with_reuse_compose_env_in_dir "${COMPOSE_DIR}" docker compose up -d
+ wait_for_https_ready "${gateway_url}/health/gateway/ready" "${ca_bundle}"
+ register_nemo_context "${context_name}" "${gateway_url}" "${ca_bundle}"
+ write_lifecycle_state compose "${REUSE_COMPOSE_PROJECT_NAME}" \
+ "target=compose" \
+ "instance_key=${INSTANCE_KEY}" \
+ "context_name=${context_name}" \
+ "gateway_url=${gateway_url}" \
+ "certificate_authority=${ca_bundle}" \
+ "workspace=${AUTHENTIK_WORKSPACE}" \
+ "compose_dir=${COMPOSE_DIR}" \
+ "compose_project_name=${REUSE_COMPOSE_PROJECT_NAME}" \
+ "compose_gateway_port=${REUSE_COMPOSE_GATEWAY_PORT}" \
+ "compose_gateway_tls_volume=${REUSE_COMPOSE_GATEWAY_TLS_VOLUME}" \
+ "compose_workload_network_name=${REUSE_COMPOSE_WORKLOAD_NETWORK_NAME}"
+
+ echo "Authentik Compose is ready: ${gateway_url}"
+ echo "NeMo context: ${context_name}"
+ echo "Compose project: ${REUSE_COMPOSE_PROJECT_NAME}"
+ echo "Lifecycle state: $(lifecycle_state_file compose "${REUSE_COMPOSE_PROJECT_NAME}")"
+ echo "Stop it with: ${SCRIPT_DIR}/run.sh down compose$(key_option_suffix)"
}
compose_down() {
local status=0
+ local state_dir
+ local state_file
+ local found_state="false"
+ local context_name
+ local state_compose_dir
+ local original_project_name="${REUSE_COMPOSE_PROJECT_NAME}"
+ local original_gateway_port="${REUSE_COMPOSE_GATEWAY_PORT}"
+ local original_gateway_tls_volume="${REUSE_COMPOSE_GATEWAY_TLS_VOLUME}"
+ local original_workload_network_name="${REUSE_COMPOSE_WORKLOAD_NETWORK_NAME}"
+
+ state_dir="$(lifecycle_state_dir)"
+ if [[ -z "${INSTANCE_KEY}" ]]; then
+ run_with_compose_env_in_dir "${COMPOSE_DIR}" docker compose down -v --remove-orphans || status="$?"
+ fi
- run_with_compose_env_in_dir "${COMPOSE_DIR}" docker compose down -v --remove-orphans || status="$?"
- run_with_reuse_compose_env_in_dir "${COMPOSE_DIR}" docker compose down -v --remove-orphans || status="$?"
+ for state_file in "${state_dir}"/compose-*.json; do
+ [[ -e "${state_file}" ]] || continue
+ state_matches_instance_key "${state_file}" || continue
+ found_state="true"
+ context_name="$(read_lifecycle_field "${state_file}" context_name)"
+ state_compose_dir="$(read_lifecycle_field "${state_file}" compose_dir)"
+ REUSE_COMPOSE_PROJECT_NAME="$(read_lifecycle_field "${state_file}" compose_project_name)"
+ REUSE_COMPOSE_GATEWAY_PORT="$(read_lifecycle_field "${state_file}" compose_gateway_port)"
+ REUSE_COMPOSE_GATEWAY_TLS_VOLUME="$(read_lifecycle_field "${state_file}" compose_gateway_tls_volume)"
+ REUSE_COMPOSE_WORKLOAD_NETWORK_NAME="$(read_lifecycle_field "${state_file}" compose_workload_network_name)"
+
+ if [[ -z "${state_compose_dir}" ]]; then
+ state_compose_dir="${COMPOSE_DIR}"
+ fi
+ echo "Stopping Authentik Compose instance: ${REUSE_COMPOSE_PROJECT_NAME}"
+ run_with_reuse_compose_env_in_dir "${state_compose_dir}" docker compose down -v --remove-orphans || status="$?"
+ delete_nemo_context "${context_name}"
+ remove_lifecycle_state_file "${state_file}"
+ done
+
+ if [[ "${found_state}" != "true" ]]; then
+ run_with_reuse_compose_env_in_dir "${COMPOSE_DIR}" docker compose down -v --remove-orphans || status="$?"
+ delete_nemo_context "$(compose_context_name)"
+ fi
+
+ REUSE_COMPOSE_PROJECT_NAME="${original_project_name}"
+ REUSE_COMPOSE_GATEWAY_PORT="${original_gateway_port}"
+ REUSE_COMPOSE_GATEWAY_TLS_VOLUME="${original_gateway_tls_volume}"
+ REUSE_COMPOSE_WORKLOAD_NETWORK_NAME="${original_workload_network_name}"
return "${status}"
}
+k8s_state_dir_for_cluster() {
+ local cluster_name="$1"
+ printf "%s/.generated/k8s/%s" "${AUTHENTIK_ROOT}" "${cluster_name}"
+}
+
+k8s_kubeconfig_file_for_cluster() {
+ local cluster_name="$1"
+ printf "%s/kubeconfig.yaml" "$(k8s_state_dir_for_cluster "${cluster_name}")"
+}
+
+k8s_ca_bundle_file_for_cluster() {
+ local cluster_name="$1"
+ printf "%s/ca.crt" "$(k8s_state_dir_for_cluster "${cluster_name}")"
+}
+
+k8s_port_forward_pid_file_for_cluster() {
+ local cluster_name="$1"
+ printf "%s/port-forward.pid" "$(k8s_state_dir_for_cluster "${cluster_name}")"
+}
+
+k8s_port_forward_log_file_for_cluster() {
+ local cluster_name="$1"
+ printf "%s/port-forward.log" "$(k8s_state_dir_for_cluster "${cluster_name}")"
+}
+
+k8s_context_for_cluster() {
+ local runtime="$1"
+ local cluster_name="$2"
+
+ case "${runtime}" in
+ kind)
+ printf "kind-%s" "${cluster_name}"
+ ;;
+ k3d)
+ printf "k3d-%s" "${cluster_name}"
+ ;;
+ *)
+ die "--runtime must be kind or k3d"
+ ;;
+ esac
+}
+
+ensure_k8s_state_dir() {
+ local cluster_name="$1"
+ local state_dir
+
+ state_dir="$(k8s_state_dir_for_cluster "${cluster_name}")"
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ printf "+ mkdir -p %q\n" "${state_dir}"
+ return
+ fi
+
+ mkdir -p "${state_dir}"
+}
+
+k8s_write_existing_kubeconfig() {
+ local cluster_name="$1"
+ local kubeconfig="$2"
+
+ case "${K8S_RUNTIME}" in
+ kind)
+ kind get kubeconfig --name "${cluster_name}" >"${kubeconfig}"
+ ;;
+ k3d)
+ k3d kubeconfig get "${cluster_name}" >"${kubeconfig}"
+ ;;
+ esac
+}
+
+k8s_create_cluster() {
+ local cluster_name="$1"
+ local kubeconfig="$2"
+
+ case "${K8S_RUNTIME}" in
+ kind)
+ run_command kind create cluster --name "${cluster_name}" --kubeconfig "${kubeconfig}" --wait 180s
+ ;;
+ k3d)
+ run_command k3d cluster create "${cluster_name}" \
+ --wait \
+ --agents 0 \
+ --k3s-arg "--disable=traefik@server:0" \
+ --kubeconfig-update-default=false
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ print_command k3d kubeconfig get "${cluster_name}"
+ printf "+ write %q\n" "${kubeconfig}"
+ else
+ k3d kubeconfig get "${cluster_name}" >"${kubeconfig}"
+ fi
+ ;;
+ esac
+}
+
+k8s_update_default_kubeconfig() {
+ local cluster_name="$1"
+ local context="$2"
+
+ case "${K8S_RUNTIME}" in
+ kind)
+ run_command kind export kubeconfig --name "${cluster_name}"
+ ;;
+ k3d)
+ run_command k3d kubeconfig merge "${cluster_name}" \
+ --kubeconfig-merge-default \
+ --kubeconfig-switch-context
+ ;;
+ esac
+ run_command kubectl config use-context "${context}"
+}
+
+k8s_ensure_cluster() {
+ local cluster_name="$1"
+ local kubeconfig="$2"
+
+ ensure_k8s_state_dir "${cluster_name}"
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ case "${K8S_RUNTIME}" in
+ kind)
+ print_command kind get kubeconfig --name "${cluster_name}"
+ ;;
+ k3d)
+ print_command k3d kubeconfig get "${cluster_name}"
+ ;;
+ esac
+ printf "+ if cluster is missing:\n"
+ k8s_create_cluster "${cluster_name}" "${kubeconfig}"
+ return
+ fi
+
+ if k8s_write_existing_kubeconfig "${cluster_name}" "${kubeconfig}" 2>/dev/null; then
+ echo "Using existing ${K8S_RUNTIME} cluster: ${cluster_name}"
+ return
+ fi
+
+ echo "Creating ${K8S_RUNTIME} cluster: ${cluster_name}"
+ k8s_create_cluster "${cluster_name}" "${kubeconfig}"
+}
+
+k8s_load_image() {
+ local cluster_name="$1"
+
+ if [[ "${K8S_SKIP_IMAGE_LOAD}" == "1" ]]; then
+ echo "Skipping Kubernetes image load for: $(image_ref)"
+ return
+ fi
+
+ case "${K8S_RUNTIME}" in
+ kind)
+ run_command kind load docker-image "$(image_ref)" --name "${cluster_name}"
+ ;;
+ k3d)
+ run_command k3d image import "$(image_ref)" -c "${cluster_name}"
+ ;;
+ esac
+}
+
+k8s_helm_command() {
+ local context="$1"
+ local kubeconfig="$2"
+ shift 2
+
+ run_in_repo helm --kubeconfig "${kubeconfig}" --kube-context "${context}" "$@"
+}
+
+k8s_kubectl_command() {
+ local context="$1"
+ local kubeconfig="$2"
+ shift 2
+
+ run_command kubectl --kubeconfig "${kubeconfig}" --context "${context}" "$@"
+}
+
+k8s_helm_install() {
+ local cluster_name="$1"
+ local context="$2"
+ local kubeconfig="$3"
+ local image
+ local registry
+ local tag
+ local workload_token_private_key
+ local -a args
+
+ image="$(image_ref)"
+ registry="${image%/nmp-api:*}"
+ tag="${image##*:}"
+ workload_token_private_key="$(workload_token_private_key_file)"
+
+ run_in_repo helm repo add nvidia https://helm.ngc.nvidia.com/nvidia --force-update
+ run_in_repo helm repo add authentik https://charts.goauthentik.io --force-update
+ run_in_repo helm dependency build k8s/helm
+ run_in_repo helm dependency build contrib/auth/authentik/helm
+
+ args=(
+ upgrade
+ --install
+ "${HELM_RELEASE}"
+ "${AUTHENTIK_ROOT}/helm"
+ --namespace
+ "${HELM_NAMESPACE}"
+ --create-namespace
+ --wait
+ --wait-for-jobs
+ --timeout
+ "${HELM_WAIT_TIMEOUT}"
+ --set
+ "nemo-platform.api.image.repository=${registry}/nmp-api"
+ --set
+ "nemo-platform.api.image.tag=${tag}"
+ --set
+ "nemo-platform.core.image.repository=${registry}/nmp-api"
+ --set
+ "nemo-platform.core.image.tag=${tag}"
+ --set-string
+ "nemo-platform.platformConfig.platform.image_registry=${registry}"
+ --set-string
+ "nemo-platform.platformConfig.platform.image_tag=${tag}"
+ --set-string
+ "nemo-platform.platformConfig.auth.access_keys.enabled=true"
+ --set-string
+ "nemo-platform.authentikPublicGateway.port=${K8S_GATEWAY_PORT}"
+ --set-file
+ "workloadTokenSigningKey.privateKeyPem=${workload_token_private_key}"
+ )
+ if [[ -n "${K8S_NGC_EXISTING_SECRET}" ]]; then
+ args+=(--set-string "nemo-platform.existingSecret=${K8S_NGC_EXISTING_SECRET}")
+ fi
+ if [[ -n "${K8S_IMAGE_PULL_SECRET}" ]]; then
+ args+=(--set-string "nemo-platform.imagePullSecrets[0].name=${K8S_IMAGE_PULL_SECRET}")
+ fi
+
+ echo "Installing Authentik Kubernetes demo into ${cluster_name}/${HELM_NAMESPACE}"
+ k8s_helm_command "${context}" "${kubeconfig}" "${args[@]}"
+}
+
+k8s_wait_for_authentik() {
+ local context="$1"
+ local kubeconfig="$2"
+ local deployment
+
+ for deployment in authentik-server authentik-worker nemo-platform-api nemo-platform-envoy; do
+ k8s_kubectl_command "${context}" "${kubeconfig}" \
+ -n "${HELM_NAMESPACE}" rollout status "deploy/${deployment}" --timeout=240s
+ done
+ k8s_kubectl_command "${context}" "${kubeconfig}" \
+ -n "${HELM_NAMESPACE}" rollout status statefulset/shared-postgresql --timeout=240s
+}
+
+k8s_write_ca_bundle() {
+ local context="$1"
+ local kubeconfig="$2"
+ local ca_bundle="$3"
+ local encoded
+
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ print_command kubectl --kubeconfig "${kubeconfig}" --context "${context}" \
+ -n "${HELM_NAMESPACE}" get secret nemo-platform-envoy-tls -o "jsonpath={.data.ca\\.crt}"
+ printf "+ write %q\n" "${ca_bundle}"
+ return
+ fi
+
+ encoded="$(
+ kubectl --kubeconfig "${kubeconfig}" --context "${context}" \
+ -n "${HELM_NAMESPACE}" get secret nemo-platform-envoy-tls -o "jsonpath={.data.ca\\.crt}"
+ )"
+ if [[ -z "${encoded}" ]]; then
+ die "secret nemo-platform-envoy-tls in ${HELM_NAMESPACE} has no ca.crt entry"
+ fi
+ if printf "%s" "${encoded}" | base64 --decode >"${ca_bundle}" 2>/dev/null; then
+ return
+ fi
+ if printf "%s" "${encoded}" | base64 -D >"${ca_bundle}" 2>/dev/null; then
+ return
+ fi
+ die "failed to decode Kubernetes gateway CA bundle"
+}
+
+stop_k8s_port_forward_for_cluster() {
+ local cluster_name="$1"
+ local pid_file
+ local pid
+
+ pid_file="$(k8s_port_forward_pid_file_for_cluster "${cluster_name}")"
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ printf "+ stop Kubernetes gateway port-forward recorded in %q if running\n" "${pid_file}"
+ return
+ fi
+ if [[ ! -f "${pid_file}" ]]; then
+ return
+ fi
+
+ pid="$(<"${pid_file}")"
+ if k8s_port_forward_pid_is_running "${pid}"; then
+ echo "Stopping Kubernetes gateway port-forward: ${pid}"
+ kill "${pid}" 2>/dev/null || true
+ wait "${pid}" 2>/dev/null || true
+ fi
+ rm -f "${pid_file}"
+}
+
+k8s_port_forward_pid_is_running() {
+ local pid="$1"
+ local expected_port="${2:-}"
+ local args
+
+ [[ "${pid}" =~ ^[0-9]+$ ]] || return 1
+ kill -0 "${pid}" 2>/dev/null || return 1
+ args="$(ps -p "${pid}" -o args= 2>/dev/null || true)"
+ [[ "${args}" == *"kubectl"* ]] || return 1
+ [[ "${args}" == *"port-forward"* ]] || return 1
+ [[ "${args}" == *"svc/nemo-platform-envoy"* ]] || return 1
+ if [[ -n "${expected_port}" ]]; then
+ [[ "${args}" == *"${expected_port}:8080"* ]] || return 1
+ fi
+}
+
+show_k8s_port_forward_log_tail() {
+ local log_file="$1"
+
+ if [[ ! -f "${log_file}" ]]; then
+ echo "Kubernetes gateway port-forward log not found: ${log_file}" >&2
+ return
+ fi
+ if [[ ! -s "${log_file}" ]]; then
+ echo "Kubernetes gateway port-forward log is empty: ${log_file}" >&2
+ return
+ fi
+
+ echo "Kubernetes gateway port-forward log tail (${log_file}):" >&2
+ tail -n 40 "${log_file}" >&2 || true
+}
+
+k8s_wait_for_port_forward_ready() {
+ local gateway_url="$1"
+ local ca_bundle="$2"
+ local log_file="$3"
+
+ if wait_for_https_ready "${gateway_url}/health/gateway/ready" "${ca_bundle}" 30; then
+ return
+ fi
+ show_k8s_port_forward_log_tail "${log_file}"
+ die "timed out waiting for Kubernetes gateway port-forward readiness"
+}
+
+k8s_start_port_forward() {
+ local cluster_name="$1"
+ local context="$2"
+ local kubeconfig="$3"
+ local ca_bundle="$4"
+ local pid_file
+ local log_file
+ local pid=""
+ local gateway_url="https://127.0.0.1:${K8S_GATEWAY_PORT}"
+
+ pid_file="$(k8s_port_forward_pid_file_for_cluster "${cluster_name}")"
+ log_file="$(k8s_port_forward_log_file_for_cluster "${cluster_name}")"
+
+ if [[ "${DRY_RUN}" == "true" ]]; then
+ printf "+ nohup "
+ quote_args kubectl --kubeconfig "${kubeconfig}" --context "${context}" \
+ -n "${HELM_NAMESPACE}" port-forward svc/nemo-platform-envoy "${K8S_GATEWAY_PORT}:8080"
+ printf "> %q 2>&1 &\n" "${log_file}"
+ printf "+ write %q\n" "${pid_file}"
+ wait_for_https_ready "${gateway_url}/health/gateway/ready" "${ca_bundle}" 30
+ return
+ fi
+
+ if [[ -f "${pid_file}" ]]; then
+ pid="$(<"${pid_file}")"
+ fi
+ if k8s_port_forward_pid_is_running "${pid}" "${K8S_GATEWAY_PORT}"; then
+ echo "Using existing Kubernetes gateway port-forward: ${pid}"
+ k8s_wait_for_port_forward_ready "${gateway_url}" "${ca_bundle}" "${log_file}"
+ return
+ fi
+ if [[ -n "${pid}" ]]; then
+ stop_k8s_port_forward_for_cluster "${cluster_name}"
+ fi
+
+ nohup kubectl --kubeconfig "${kubeconfig}" --context "${context}" \
+ -n "${HELM_NAMESPACE}" port-forward svc/nemo-platform-envoy "${K8S_GATEWAY_PORT}:8080" \
+ >"${log_file}" 2>&1 &
+ printf "%s\n" "$!" >"${pid_file}"
+ k8s_wait_for_port_forward_ready "${gateway_url}" "${ca_bundle}" "${log_file}"
+}
+
+k8s_up() {
+ local cluster_name
+ local context
+ local nemo_context
+ local kubeconfig
+ local ca_bundle
+ local gateway_url="https://127.0.0.1:${K8S_GATEWAY_PORT}"
+
+ validate_k8s_runtime
+ ensure_workload_token_private_key
+ cluster_name="${K8S_CLUSTER_NAME:-${REUSE_K8S_CLUSTER_NAME}}"
+ K8S_CLUSTER_NAME="${cluster_name}"
+ context="$(k8s_context_for_cluster "${K8S_RUNTIME}" "${cluster_name}")"
+ nemo_context="$(k8s_context_name)"
+ kubeconfig="$(k8s_kubeconfig_file_for_cluster "${cluster_name}")"
+ ca_bundle="$(k8s_ca_bundle_file_for_cluster "${cluster_name}")"
+
+ if [[ "${IMAGE_SELECTED}" == "true" ]]; then
+ echo "Using prebuilt auth-idp Kubernetes image: $(image_ref)"
+ elif [[ "${K8S_SKIP_IMAGE_LOAD}" == "1" ]]; then
+ echo "Reusing Kubernetes cluster without rebuilding or loading image: $(image_ref)"
+ else
+ build_default_test_image
+ fi
+
+ k8s_ensure_cluster "${cluster_name}" "${kubeconfig}"
+ k8s_update_default_kubeconfig "${cluster_name}" "${context}"
+ k8s_load_image "${cluster_name}"
+ k8s_helm_install "${cluster_name}" "${context}" "${kubeconfig}"
+ k8s_wait_for_authentik "${context}" "${kubeconfig}"
+ k8s_write_ca_bundle "${context}" "${kubeconfig}" "${ca_bundle}"
+ k8s_start_port_forward "${cluster_name}" "${context}" "${kubeconfig}" "${ca_bundle}"
+ register_nemo_context "${nemo_context}" "${gateway_url}" "${ca_bundle}"
+ write_lifecycle_state k8s "${K8S_RUNTIME}-${cluster_name}" \
+ "target=k8s" \
+ "instance_key=${INSTANCE_KEY}" \
+ "context_name=${nemo_context}" \
+ "gateway_url=${gateway_url}" \
+ "certificate_authority=${ca_bundle}" \
+ "workspace=${AUTHENTIK_WORKSPACE}" \
+ "runtime=${K8S_RUNTIME}" \
+ "cluster_name=${cluster_name}" \
+ "kubernetes_context=${context}" \
+ "kubeconfig=${kubeconfig}" \
+ "namespace=${HELM_NAMESPACE}" \
+ "helm_release=${HELM_RELEASE}" \
+ "gateway_port=${K8S_GATEWAY_PORT}" \
+ "port_forward_pid_file=$(k8s_port_forward_pid_file_for_cluster "${cluster_name}")"
+
+ echo "Authentik Kubernetes is ready: ${gateway_url}"
+ echo "NeMo context: ${nemo_context}"
+ echo "Cluster: ${cluster_name}"
+ echo "Kubernetes context: ${context}"
+ echo "Kubeconfig: ${kubeconfig}"
+ echo "Gateway CA bundle: ${ca_bundle}"
+ echo "Lifecycle state: $(lifecycle_state_file k8s "${K8S_RUNTIME}-${cluster_name}")"
+ echo "Stop it with: ${SCRIPT_DIR}/run.sh down k8s$(key_option_suffix)"
+}
+
delete_reuse_k8s_cluster() {
local cluster_name="${K8S_CLUSTER_NAME:-${REUSE_K8S_CLUSTER_NAME}}"
local -a delete_command
@@ -500,11 +1308,62 @@ delete_reuse_k8s_cluster() {
fi
}
+k8s_down() {
+ local status=0
+ local state_dir
+ local state_file
+ local found_state="false"
+ local context_name
+ local original_runtime="${K8S_RUNTIME}"
+ local original_cluster_name="${K8S_CLUSTER_NAME}"
+
+ state_dir="$(lifecycle_state_dir)"
+ for state_file in "${state_dir}"/k8s-*.json; do
+ [[ -e "${state_file}" ]] || continue
+ state_matches_instance_key "${state_file}" || continue
+ found_state="true"
+ context_name="$(read_lifecycle_field "${state_file}" context_name)"
+ K8S_RUNTIME="$(read_lifecycle_field "${state_file}" runtime)"
+ K8S_CLUSTER_NAME="$(read_lifecycle_field "${state_file}" cluster_name)"
+
+ echo "Stopping Authentik Kubernetes instance: ${K8S_RUNTIME}/${K8S_CLUSTER_NAME}"
+ stop_k8s_port_forward_for_cluster "${K8S_CLUSTER_NAME}" || status="$?"
+ delete_reuse_k8s_cluster || status="$?"
+ delete_nemo_context "${context_name}"
+ remove_lifecycle_state_file "${state_file}"
+ done
+
+ if [[ "${found_state}" != "true" ]]; then
+ K8S_CLUSTER_NAME="${K8S_CLUSTER_NAME:-${REUSE_K8S_CLUSTER_NAME}}"
+ stop_k8s_port_forward_for_cluster "${K8S_CLUSTER_NAME}" || status="$?"
+ delete_reuse_k8s_cluster || status="$?"
+ delete_nemo_context "$(k8s_context_name)"
+ fi
+
+ K8S_RUNTIME="${original_runtime}"
+ K8S_CLUSTER_NAME="${original_cluster_name}"
+ return "${status}"
+}
+
down() {
+ local target="${1:-all}"
local status=0
- compose_down || status="$?"
- delete_reuse_k8s_cluster || status="$?"
+ case "${target}" in
+ compose)
+ compose_down || status="$?"
+ ;;
+ k8s)
+ k8s_down || status="$?"
+ ;;
+ all)
+ compose_down || status="$?"
+ k8s_down || status="$?"
+ ;;
+ *)
+ die "down target must be compose, k8s, or all"
+ ;;
+ esac
return "${status}"
}
@@ -622,6 +1481,7 @@ run_k8s_tests() {
"E2E_SERVICES_LOG_DIR=${diagnostics}" \
"NMP_AUTHENTIK_K8S_LOG_DIR=${k8s_diagnostics}" \
"NMP_AUTHENTIK_K8S_HELM_RELEASE=${HELM_RELEASE}" \
+ "NMP_AUTHENTIK_K8S_HELM_WAIT_TIMEOUT=${HELM_WAIT_TIMEOUT}" \
"NMP_AUTHENTIK_K8S_NAMESPACE=${HELM_NAMESPACE}" \
"NMP_AUTHENTIK_K8S_RUNTIME=${K8S_RUNTIME}" \
"NMP_AUTHENTIK_K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME}" \
@@ -642,6 +1502,7 @@ run_k8s_tests() {
"E2E_SERVICES_LOG_DIR=${diagnostics}" \
"NMP_AUTHENTIK_K8S_LOG_DIR=${k8s_diagnostics}" \
"NMP_AUTHENTIK_K8S_HELM_RELEASE=${HELM_RELEASE}" \
+ "NMP_AUTHENTIK_K8S_HELM_WAIT_TIMEOUT=${HELM_WAIT_TIMEOUT}" \
"NMP_AUTHENTIK_K8S_NAMESPACE=${HELM_NAMESPACE}" \
"NMP_AUTHENTIK_K8S_RUNTIME=${K8S_RUNTIME}" \
"NMP_AUTHENTIK_K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME}" \
@@ -665,7 +1526,40 @@ fi
while [[ $# -gt 0 ]]; do
case "$1" in
- run-local | down | prepare-local | compose | render-blueprint | k8s)
+ up | test)
+ if [[ -n "${ACTION}" ]]; then
+ die "only one action can be specified"
+ fi
+ ACTION="$1"
+ shift
+ [[ $# -ge 1 ]] || die "${ACTION} requires a target: compose or k8s"
+ case "$1" in
+ compose | k8s)
+ TARGET="$1"
+ shift
+ ;;
+ *)
+ die "${ACTION} target must be compose or k8s"
+ ;;
+ esac
+ ;;
+ down | clean)
+ if [[ -n "${ACTION}" ]]; then
+ die "only one action can be specified"
+ fi
+ ACTION="$1"
+ TARGET="all"
+ shift
+ if [[ $# -gt 0 ]]; then
+ case "$1" in
+ compose | k8s | all)
+ TARGET="$1"
+ shift
+ ;;
+ esac
+ fi
+ ;;
+ prepare-local | render-blueprint)
if [[ -n "${ACTION}" ]]; then
die "only one action can be specified"
fi
@@ -703,6 +1597,15 @@ while [[ $# -gt 0 ]]; do
K8S_SKIP_IMAGE_LOAD_SET="true"
shift
;;
+ --key)
+ [[ $# -ge 2 ]] || die "--key requires a value"
+ INSTANCE_KEY="$2"
+ shift 2
+ ;;
+ --key=*)
+ INSTANCE_KEY="${1#*=}"
+ shift
+ ;;
--compose-dir)
[[ $# -ge 2 ]] || die "--compose-dir requires a value"
COMPOSE_DIR="$2"
@@ -727,8 +1630,17 @@ if [[ -z "${ACTION}" ]]; then
die "missing action"
fi
+if [[ -n "${INSTANCE_KEY}" &&
+ "${ACTION}" != "up" &&
+ "${ACTION}" != "down" &&
+ "${ACTION}" != "clean" ]]; then
+ die "--key is only valid with up, down, or clean"
+fi
+
+configure_instance_defaults
+
if [[ "${REUSE_SET}" == "true" ]]; then
- case "${ACTION}" in
+ case "${TARGET}" in
compose)
TEST_LIFECYCLE="reuse"
;;
@@ -745,7 +1657,22 @@ if [[ "${REUSE_SET}" == "true" ]]; then
esac
fi
-if [[ "${ACTION}" == "k8s" && "${K8S_REUSE_CLUSTER}" == "1" && -z "${K8S_CLUSTER_NAME}" ]]; then
+if [[ "${ACTION}" == "up" ]]; then
+ case "${TARGET}" in
+ compose)
+ TEST_LIFECYCLE="reuse"
+ ;;
+ k8s)
+ K8S_REUSE_CLUSTER="1"
+ K8S_KEEP_CLUSTER="1"
+ if [[ -z "${K8S_CLUSTER_NAME}" ]]; then
+ K8S_CLUSTER_NAME="${REUSE_K8S_CLUSTER_NAME}"
+ fi
+ ;;
+ esac
+fi
+
+if [[ "${TARGET}" == "k8s" && "${K8S_REUSE_CLUSTER}" == "1" && -z "${K8S_CLUSTER_NAME}" ]]; then
K8S_CLUSTER_NAME="${REUSE_K8S_CLUSTER_NAME}"
fi
@@ -753,25 +1680,25 @@ if [[ -z "${IMAGE_REGISTRY}" || -z "${BAKE_TAG}" ]]; then
die "image registry and tag must be non-empty"
fi
-if [[ "${ACTION}" != "compose" && "${ACTION}" != "k8s" ]]; then
+if [[ "${TARGET}" != "compose" && "${TARGET}" != "k8s" ]]; then
if [[ "${TEST_PLATFORM_SET}" == "true" ]]; then
die "--platform is only valid with the compose or k8s action"
fi
fi
-if [[ "${ACTION}" != "k8s" && "${ACTION}" != "down" ]]; then
+if [[ "${TARGET}" != "k8s" && "${ACTION}" != "down" && "${ACTION}" != "clean" ]]; then
if [[ "${K8S_RUNTIME_SET}" == "true" ]]; then
- die "--runtime is only valid with k8s or down"
+ die "--runtime is only valid with k8s, down, or clean"
fi
fi
-if [[ "${ACTION}" != "k8s" ]]; then
+if [[ "${TARGET}" != "k8s" ]]; then
if [[ "${K8S_SKIP_IMAGE_LOAD_SET}" == "true" ]]; then
die "--skip-image-load is only valid with k8s"
fi
fi
-if [[ "${ACTION}" == "k8s" &&
+if [[ "${TARGET}" == "k8s" &&
"${K8S_SKIP_IMAGE_LOAD}" == "1" &&
"${K8S_REUSE_CLUSTER}" != "1" &&
"${IMAGE_SELECTED}" != "true" ]]; then
@@ -779,27 +1706,43 @@ if [[ "${ACTION}" == "k8s" &&
"--image, or a reused cluster via --reuse"
fi
-if [[ "${ACTION}" != "run-local" && "${ACTION}" != "down" && "${COMPOSE_DIR_SET}" == "true" ]]; then
- die "--compose-dir is only valid with run-local or down"
+if [[ "${ACTION}" != "down" &&
+ "${ACTION}" != "clean" &&
+ "${TARGET}" != "compose" &&
+ "${COMPOSE_DIR_SET}" == "true" ]]; then
+ die "--compose-dir is only valid with down, clean, or compose"
fi
+configure_k8s_gateway_port
+
case "${ACTION}" in
- run-local)
- run_local
+ up)
+ case "${TARGET}" in
+ compose)
+ compose_up
+ ;;
+ k8s)
+ k8s_up
+ ;;
+ esac
+ ;;
+ test)
+ case "${TARGET}" in
+ compose)
+ run_tests
+ ;;
+ k8s)
+ run_k8s_tests
+ ;;
+ esac
;;
- down)
- down
+ down | clean)
+ down "${TARGET}"
;;
prepare-local)
prepare_local
;;
- compose)
- run_tests
- ;;
render-blueprint)
render_blueprint
;;
- k8s)
- run_k8s_tests
- ;;
esac
diff --git a/docs/cli/configuration.mdx b/docs/cli/configuration.mdx
index 863ed77149..bf0ae04262 100644
--- a/docs/cli/configuration.mdx
+++ b/docs/cli/configuration.mdx
@@ -40,12 +40,24 @@ Set specific values:
```bash
nemo config set --base-url https://nmp.example.com
+nemo config set --context local-tls --base-url https://localhost:8443 --certificate-authority /path/to/ca.crt
nemo config set --workspace my-workspace
nemo config set --access-token -
```
+The saved cluster entry uses the `certificate_authority` key.
+
When setting an access token, you'll be prompted to enter it securely (input is hidden).
+Delete a context without deleting its referenced cluster or user records:
+
+```bash
+nemo config delete-context local-tls
+```
+
+To also remove cluster and user records that are no longer referenced by any
+context, pass `--prune-orphans`.
+
## Environment variables
Environment variables override configuration file settings. This is useful for CI/CD pipelines or temporary overrides.
diff --git a/docs/cli/connect-to-deployments.mdx b/docs/cli/connect-to-deployments.mdx
index 727dda9ac6..24c12ebbc2 100644
--- a/docs/cli/connect-to-deployments.mdx
+++ b/docs/cli/connect-to-deployments.mdx
@@ -89,6 +89,27 @@ A local platform can coexist with remote contexts:
nemo config set --context local --base-url http://localhost:8080
```
+For a local HTTPS deployment with a private certificate authority, save the CA
+bundle with the cluster:
+
+```bash
+nemo config set --context local-tls --base-url https://localhost:8443 --certificate-authority /path/to/ca.crt
+```
+
+The saved cluster entry uses the `certificate_authority` key.
+
+## Delete a context
+
+Delete a context when you no longer need it:
+
+```bash
+nemo config delete-context local-tls
+```
+
+This follows the same convention as `kubectl config delete-context`: the
+context is removed, but cluster and user records remain. To also remove
+unreferenced cluster and user records, pass `--prune-orphans`.
+
`NMP_BASE_URL` and `NMP_CURRENT_CONTEXT` override saved configuration for all contexts. If switching contexts does not change the target deployment, unset those variables or update them for the current shell.
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/device_flow.py b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/device_flow.py
index d45ed21ea5..95ad8066e2 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/device_flow.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/device_flow.py
@@ -13,7 +13,7 @@
from rich.panel import Panel
from nemo_platform_ext.auth.token_provider import refresh_token_grant
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import httpx_tls_config_from_env
console = Console()
@@ -66,15 +66,17 @@ def __init__(
token_endpoint: str,
client_id: str,
scope: str = "openid email profile",
+ certificate_authority: str | None = None,
):
self.device_authorization_endpoint = device_authorization_endpoint
self.token_endpoint = token_endpoint
self.client_id = client_id
self.scope = scope
+ self.certificate_authority = certificate_authority
async def start_device_authorization(self) -> DeviceCodeResponse:
"""Start the device authorization flow."""
- async with httpx.AsyncClient(verify=client_verify_from_env()) as client:
+ async with httpx.AsyncClient(**httpx_tls_config_from_env(self.certificate_authority)) as client:
response = await client.post(
self.device_authorization_endpoint,
data={
@@ -104,7 +106,7 @@ async def poll_for_token(
"""Poll the token endpoint until authorization is complete."""
start_time = time.time()
- async with httpx.AsyncClient(verify=client_verify_from_env()) as client:
+ async with httpx.AsyncClient(**httpx_tls_config_from_env(self.certificate_authority)) as client:
while time.time() - start_time < expires_in:
await _async_pause(interval)
@@ -154,6 +156,7 @@ async def authenticate_with_device_flow(
client_id: str,
scope: str = "openid email profile",
open_browser: bool = True,
+ certificate_authority: str | None = None,
) -> TokenResponse:
"""Perform OAuth device flow authentication.
@@ -172,6 +175,7 @@ async def authenticate_with_device_flow(
token_endpoint=token_endpoint,
client_id=client_id,
scope=scope,
+ certificate_authority=certificate_authority,
)
# Start device authorization
@@ -215,6 +219,7 @@ async def refresh_access_token(
client_id: str,
refresh_token: str,
scope: str | None = None,
+ certificate_authority: str | None = None,
) -> TokenResponse:
"""
Refresh an access token using a refresh token.
@@ -238,6 +243,7 @@ async def refresh_access_token(
client_id,
refresh_token,
scope=scope,
+ certificate_authority=certificate_authority,
)
except RuntimeError as e:
raise DeviceFlowError(str(e)) from e
@@ -258,6 +264,7 @@ def authenticate_with_password_grant(
username: str,
password: str,
scope: str = "openid profile email",
+ certificate_authority: str | None = None,
) -> TokenResponse:
"""Obtain tokens using the Resource Owner Password Credentials grant (RFC 6749).
@@ -284,7 +291,7 @@ def authenticate_with_password_grant(
"password": password,
"scope": scope,
}
- with httpx.Client(verify=client_verify_from_env()) as client:
+ with httpx.Client(**httpx_tls_config_from_env(certificate_authority)) as client:
response = client.post(token_endpoint, data=data, timeout=30.0)
if response.status_code != 200:
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/helpers.py b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/helpers.py
index 9c3281afe0..1cf3fc7232 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/helpers.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/helpers.py
@@ -25,7 +25,7 @@
import httpx
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import httpx_tls_config_from_env
DEFAULT_OAUTH_SCOPES = "openid profile email offline_access"
@@ -152,12 +152,17 @@ class NMPOIDCConfig:
workload_scope: str | None = None
-def discover_nmp_config(base_url: str, timeout: float = 10.0) -> NMPOIDCConfig:
+def discover_nmp_config(
+ base_url: str,
+ timeout: float = 10.0,
+ *,
+ certificate_authority: str | None = None,
+) -> NMPOIDCConfig:
"""Fetch OIDC configuration from the NeMo Platform auth discovery endpoint."""
response = httpx.get(
f"{base_url.rstrip('/')}/apis/auth/discovery",
timeout=timeout,
- verify=client_verify_from_env(),
+ **httpx_tls_config_from_env(certificate_authority),
)
response.raise_for_status()
data = response.json()
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/token_provider.py b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/token_provider.py
index bcc7b7c5c8..ddc2aa33de 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/token_provider.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/token_provider.py
@@ -16,7 +16,7 @@
from typing_extensions import Self
from nemo_platform_ext.auth.helpers import decode_jwt_claims
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import httpx_tls_config_from_env
logger = logging.getLogger(__name__)
@@ -45,6 +45,7 @@ def refresh_token_grant(
refresh_token: str,
*,
scope: str | None = None,
+ certificate_authority: str | None = None,
timeout: float = 30.0,
) -> dict:
"""Execute OAuth refresh_token grant and return token response JSON."""
@@ -56,7 +57,12 @@ def refresh_token_grant(
if scope:
data["scope"] = scope
- response = httpx.post(token_endpoint, data=data, timeout=timeout, verify=client_verify_from_env())
+ response = httpx.post(
+ token_endpoint,
+ data=data,
+ timeout=timeout,
+ **httpx_tls_config_from_env(certificate_authority),
+ )
if response.status_code != 200:
error_data: dict[str, str] = {}
@@ -136,6 +142,7 @@ class OIDCTokenProvider:
tokens: TokenSet = field(default_factory=lambda: TokenSet(access_token=""))
refresh_margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS
refresh_scope: str | None = None
+ certificate_authority: str | None = None
load_tokens: Callable[[], TokenSet | None] | None = None
refresh_lock: Callable[[], AbstractContextManager[None]] | None = None
on_tokens_refreshed: Callable[[TokenSet], None] | None = None
@@ -205,6 +212,7 @@ def _refresh(self, *, force: bool = False) -> None:
client_id=self.client_id,
refresh_token=self.tokens.refresh_token,
scope=self.refresh_scope,
+ certificate_authority=self.certificate_authority,
)
except TokenRefreshError as exc:
if exc.error != "invalid_grant":
@@ -228,6 +236,7 @@ def _refresh(self, *, force: bool = False) -> None:
client_id=self.client_id,
refresh_token=self.tokens.refresh_token,
scope=self.refresh_scope,
+ certificate_authority=self.certificate_authority,
)
new_access_token = token_data["access_token"]
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/workload_exchange.py b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/workload_exchange.py
index 2e9fd8a332..51637d6574 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/auth/workload_exchange.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/auth/workload_exchange.py
@@ -19,7 +19,7 @@
from nemo_platform_plugin.client.constants import WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR
from nemo_platform_ext.auth.token_provider import DEFAULT_REFRESH_MARGIN_SECONDS, TokenSet
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import httpx_tls_config_from_env
logger = logging.getLogger(__name__)
@@ -79,6 +79,7 @@ def token_exchange_grant(
subject_token: str,
audience: str | None = None,
scope: str | None = None,
+ certificate_authority: str | None = None,
timeout: float = 30.0,
) -> dict[str, object]:
"""Execute RFC 8693 token exchange and return token response JSON."""
@@ -95,7 +96,12 @@ def token_exchange_grant(
if scope:
data["scope"] = scope
- response = httpx.post(token_endpoint, data=data, timeout=timeout, verify=client_verify_from_env())
+ response = httpx.post(
+ token_endpoint,
+ data=data,
+ timeout=timeout,
+ **httpx_tls_config_from_env(certificate_authority),
+ )
if response.status_code != 200:
error_data: dict[str, object] = {}
@@ -157,6 +163,7 @@ class WorkloadTokenExchangeProvider:
subject_token_file: Path
audience: str | None = None
scope: str | None = None
+ certificate_authority: str | None = None
refresh_margin_seconds: float = DEFAULT_REFRESH_MARGIN_SECONDS
tokens: TokenSet = field(default_factory=lambda: TokenSet(access_token=""))
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
@@ -181,6 +188,7 @@ def _exchange(self) -> None:
subject_token=subject_token,
audience=self.audience,
scope=self.scope,
+ certificate_authority=self.certificate_authority,
)
access_token = _access_token_from_response(token_data)
try:
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py
index db54c6de82..39b8ac9c02 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py
@@ -90,14 +90,18 @@ def _parse_access_key_expires_in(value: str | None) -> tuple[bool, int | None]:
return True, expires_in_seconds
-def is_auth_disabled(base_url: str, timeout: float = 3.0) -> bool:
+def is_auth_disabled(base_url: str, timeout: float = 3.0, certificate_authority: str | None = None) -> bool:
"""Check whether authentication is disabled on the cluster.
Returns:
True if auth is disabled, False if enabled.
"""
try:
- return not discover_nmp_config(base_url, timeout=timeout).auth_enabled
+ return not discover_nmp_config(
+ base_url,
+ timeout=timeout,
+ certificate_authority=certificate_authority,
+ ).auth_enabled
except httpx.HTTPError as exc:
raise AuthError(f"Failed to discover auth configuration: {exc}") from exc
@@ -162,7 +166,7 @@ def ensure_valid_token(context: Context, refresh_buffer_seconds: int = 300) -> b
base_url = str(context.cluster.base_url).rstrip("/")
try:
- nmp_config = discover_nmp_config(base_url)
+ nmp_config = discover_nmp_config(base_url, certificate_authority=context.cluster.certificate_authority)
except httpx.HTTPError:
return exp_dt > now
@@ -181,6 +185,7 @@ def ensure_valid_token(context: Context, refresh_buffer_seconds: int = 300) -> b
),
refresh_scope=effective_scope,
refresh_margin_seconds=float(refresh_buffer_seconds),
+ certificate_authority=context.cluster.certificate_authority,
)
provider.force_refresh()
@@ -227,11 +232,12 @@ def _login_with_oidc(
console = Console()
context = cli_context.get_sdk_context()
base_url = str(context.cluster.base_url).rstrip("/")
+ certificate_authority = context.cluster.certificate_authority
console.print(f"\nDiscovering auth configuration from {base_url}...")
try:
- oidc_config = discover_nmp_config(base_url)
+ oidc_config = discover_nmp_config(base_url, certificate_authority=certificate_authority)
except httpx.HTTPError as exc:
raise AuthError(f"Failed to discover auth configuration: {exc}") from exc
@@ -302,6 +308,7 @@ def _login_with_oidc(
username=login_username,
password=login_password,
scope=effective_scope,
+ certificate_authority=certificate_authority,
)
except DeviceFlowError as exc:
raise AuthError(f"Authentication failed: {exc}") from exc
@@ -317,6 +324,7 @@ def _login_with_oidc(
client_id=client_id,
scope=effective_scope,
open_browser=not no_browser,
+ certificate_authority=certificate_authority,
)
)
except DeviceFlowError as exc:
@@ -539,7 +547,7 @@ def login(
base_url = str(context.cluster.base_url).rstrip("/")
try:
- oidc_config = discover_nmp_config(base_url)
+ oidc_config = discover_nmp_config(base_url, certificate_authority=context.cluster.certificate_authority)
oidc_login_configured = bool(oidc_config.token_endpoint and oidc_config.client_id)
if oidc_login_configured:
raise AuthError(
@@ -603,7 +611,7 @@ def logout(ctx: typer.Context) -> None:
base_url = str(context.cluster.base_url).rstrip("/")
try:
- if is_auth_disabled(base_url) is True:
+ if is_auth_disabled(base_url, certificate_authority=context.cluster.certificate_authority) is True:
console.print("[yellow]Authentication is disabled on this cluster — nothing to log out from.[/]")
return
except AuthError as exc:
@@ -741,7 +749,7 @@ def refresh(ctx: typer.Context) -> None:
# Fetch client_id from cluster discovery
base_url = str(context.cluster.base_url).rstrip("/")
try:
- oidc_config = discover_nmp_config(base_url)
+ oidc_config = discover_nmp_config(base_url, certificate_authority=context.cluster.certificate_authority)
except httpx.HTTPError as e:
raise AuthError(f"Failed to discover auth configuration: {e}") from e
@@ -760,6 +768,7 @@ def refresh(ctx: typer.Context) -> None:
context.user.refresh_token.get_secret_value(),
),
refresh_scope=effective_scope,
+ certificate_authority=context.cluster.certificate_authority,
)
try:
@@ -987,7 +996,7 @@ def status(ctx: typer.Context) -> None:
base_url = str(context.cluster.base_url).rstrip("/")
auth_discovery_error: AuthError | None = None
try:
- auth_disabled = is_auth_disabled(base_url)
+ auth_disabled = is_auth_disabled(base_url, certificate_authority=context.cluster.certificate_authority)
except AuthError as exc:
logger.debug("Failed to discover auth configuration during status", exc_info=True)
auth_discovery_error = exc
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config.py
index bc4d43d91e..7aa10783cc 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config.py
@@ -96,6 +96,13 @@ def set_config(
str | None,
typer.Option("--base-url", help="NeMo Platform API base URL"),
] = None,
+ certificate_authority: Annotated[
+ str | None,
+ typer.Option(
+ "--certificate-authority",
+ help="Path to a PEM certificate authority bundle for the cluster",
+ ),
+ ] = None,
api_key: Annotated[
str | None,
typer.Option("--api-key", help="API key for authentication", hide_input=True),
@@ -143,6 +150,7 @@ def set_config(
nemo config set --base-url https://api.example.com
nemo config set --context staging --base-url https://nmp.staging.example.com
nemo config set --context production --base-url https://nmp.example.com --activate
+ nemo config set --context local-tls --base-url https://localhost:8443 --certificate-authority ./ca.crt
nemo config set --workspace my-workspace --output-format json
nemo config set --api-key YOUR_API_KEY
nemo config set --api-key - # prompts for API key securely
@@ -174,7 +182,16 @@ def set_config(
# Check if any options were provided
has_options = any(
- [base_url, api_key, access_token, workspace, output_format, timestamp_format, truncate is not None]
+ [
+ base_url,
+ certificate_authority is not None,
+ api_key,
+ access_token,
+ workspace,
+ output_format,
+ timestamp_format,
+ truncate is not None,
+ ]
)
if not has_options and not activate:
@@ -192,6 +209,8 @@ def set_config(
params: ConfigParams = {}
if base_url:
params["base_url"] = base_url
+ if certificate_authority is not None:
+ params["certificate_authority"] = certificate_authority
token = access_token or api_key
if token:
params["access_token"] = token
@@ -246,6 +265,35 @@ def use_context(
typer.echo(f"Switched to context '{context_name}'")
+@app.command("delete-context")
+@handle_errors
+def delete_context(
+ context_name: Annotated[str, typer.Argument(help="Context name to delete", autocompletion=_complete_context_names)],
+ prune_orphans: Annotated[
+ bool,
+ typer.Option(
+ "--prune-orphans",
+ help="Also delete cluster and user records no longer referenced by any context",
+ ),
+ ] = False,
+) -> None:
+ """Delete a context from the configuration file.
+
+ By default this follows kubectl's convention and deletes only the context;
+ referenced cluster and user records are kept.
+ """
+ from nemo_platform_ext.config.config import Config
+
+ result = Config.delete_context(context_name, prune_orphans=prune_orphans)
+ typer.echo(f"Deleted context '{result.context_name}'")
+ if result.current_context_cleared:
+ typer.echo("Current context cleared")
+ if result.pruned_clusters:
+ typer.echo(f"Pruned clusters: {', '.join(result.pruned_clusters)}")
+ if result.pruned_users:
+ typer.echo(f"Pruned users: {', '.join(result.pruned_users)}")
+
+
@app.command("view")
@handle_errors
def view_config(
@@ -325,6 +373,7 @@ def view_config(
"set": 1,
"current-context": 2,
"use-context": 3,
+ "delete-context": 4,
}
app.registered_commands.sort(
key=lambda command: _CONFIG_COMMAND_ORDER.get(command.name or "", len(_CONFIG_COMMAND_ORDER))
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config_help.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config_help.py
index 847aa4b30c..673d2e132e 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config_help.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/config_help.py
@@ -12,8 +12,12 @@
# Keep contexts separate.
nemo config set --context staging --base-url https://nmp.staging.example.com
nemo config set --context production --base-url https://nmp.example.com --activate
+# Configure a local TLS endpoint.
+nemo config set --context local-tls --base-url https://localhost:8443 --certificate-authority /path/to/ca.crt
# Switch the current context and inspect its configuration.
nemo config use-context staging
nemo config view
+# Delete a context without deleting its cluster or user records.
+nemo config delete-context staging
NMP_BASE_URL and NMP_CURRENT_CONTEXT override saved configuration."""
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py
index f62b66e0df..0fb313461d 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py
@@ -47,7 +47,7 @@
from nemo_platform_ext.cli.docker_preflight import DOCKER_PREFLIGHT_MESSAGE, require_docker_for_default_local
from nemo_platform_ext.cli.telemetry import emit
from nemo_platform_ext.cli.telemetry.events import OnboardingStepEvent, TaskStatusEnum
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import HttpxTLSConfig, httpx_tls_config_from_env
from nemo_platform_ext.config.config import Config
from nemo_platform_ext.config.models import DEFAULT_BASE_URL, ConfigFile, ConfigParams, LocalServicesConfig, NoAuthUser
from nemo_platform_ext.local.install import services_extra_install_command
@@ -320,11 +320,11 @@ def _check_platform_reachable(base_url: str, timeout: float = 5.0) -> bool:
Local ``nemo services run`` publishes ``/status``. Hosted deployments may
only expose ``/cluster-info`` on ingress, so try both.
"""
- verify = client_verify_from_env()
+ tls_config = httpx_tls_config_from_env()
root = base_url.rstrip("/")
for path in _PLATFORM_REACHABILITY_PATHS:
try:
- resp = httpx.get(f"{root}{path}", timeout=timeout, verify=verify)
+ resp = httpx.get(f"{root}{path}", timeout=timeout, **tls_config)
if resp.status_code == 200:
return True
except Exception:
@@ -463,10 +463,10 @@ def _platform_request_headers(cli_context: CLIContext) -> dict[str, str] | None:
return {key: value for key, value in headers.items() if isinstance(key, str) and isinstance(value, str)}
-def _hosted_platform_without_status(base_url: str, *, timeout: float, verify: str | bool) -> bool:
+def _hosted_platform_without_status(base_url: str, *, timeout: float, tls_config: HttpxTLSConfig) -> bool:
"""Return True when ``/cluster-info`` confirms a hosted platform that omits ``/status``."""
try:
- resp = httpx.get(f"{base_url.rstrip('/')}/cluster-info", timeout=timeout, verify=verify)
+ resp = httpx.get(f"{base_url.rstrip('/')}/cluster-info", timeout=timeout, **tls_config)
except Exception:
return False
return resp.status_code == 200
@@ -482,13 +482,13 @@ def _check_controller_health(base_url: str, timeout: float = 5.0) -> tuple[bool,
If ``controllers.status`` is empty on the first call (startup timing race),
waits ``_CONTROLLER_HEALTH_RETRY_DELAY`` seconds and retries once.
"""
- verify = client_verify_from_env()
+ tls_config = httpx_tls_config_from_env()
root = base_url.rstrip("/")
for attempt in range(2):
try:
- resp = httpx.get(f"{root}/status", timeout=timeout, verify=verify)
+ resp = httpx.get(f"{root}/status", timeout=timeout, **tls_config)
if resp.status_code == 404:
- if _hosted_platform_without_status(root, timeout=timeout, verify=verify):
+ if _hosted_platform_without_status(root, timeout=timeout, tls_config=tls_config):
return True, "Hosted deployment does not publish /status."
return False, "Unexpected status 404 from /status endpoint."
if resp.status_code != 200:
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py b/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py
index 90d6daa8c4..6457a424c8 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/client/enhanced.py
@@ -148,7 +148,7 @@ def __init__(
"""
env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL")
bootstrap_base_url = base_url if base_url is not None else env_base_url
-
+ client_verify = client_verify_from_env()
should_bootstrap = _should_bootstrap_config(
http_client=http_client,
base_url=base_url,
@@ -176,6 +176,7 @@ def __init__(
):
raise TypeError("Expected httpx.Client from sync client factory")
http_client = client_init_kwargs.http_client
+ client_verify = client_init_kwargs.client_verify
except Exception as e:
raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e
@@ -185,7 +186,6 @@ def __init__(
if base_url is None:
raise RuntimeError("NeMoPlatform client initialization failed: base_url is required")
- client_verify = client_verify_from_env()
if http_client is None and client_verify is not True:
http_client = DefaultHttpxClient(verify=client_verify)
@@ -383,7 +383,7 @@ async def main() -> None:
"""
env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL")
bootstrap_base_url = base_url if base_url is not None else env_base_url
-
+ client_verify = client_verify_from_env()
should_bootstrap = _should_bootstrap_config(
http_client=http_client,
base_url=base_url,
@@ -411,6 +411,7 @@ async def main() -> None:
):
raise TypeError("Expected httpx.AsyncClient from async client factory")
http_client = client_init_kwargs.http_client
+ client_verify = client_init_kwargs.client_verify
except Exception as e:
raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e
@@ -420,7 +421,6 @@ async def main() -> None:
if base_url is None:
raise RuntimeError("NeMoPlatform client initialization failed: base_url is required")
- client_verify = client_verify_from_env()
if http_client is None and client_verify is not True:
http_client = DefaultAsyncHttpxClient(verify=client_verify)
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/client/factory.py b/packages/nemo_platform_ext/src/nemo_platform_ext/client/factory.py
index 64c001c9ac..0ac320d9ed 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/client/factory.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/client/factory.py
@@ -56,7 +56,7 @@
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Callable, Mapping, Protocol
+from typing import Any, Callable, Literal, Mapping, Protocol
import httpx
from nemo_platform import (
@@ -110,6 +110,7 @@ class ClientInitConfig:
workspace: str | None
default_headers: Mapping[str, str] | None = None
http_client: httpx.Client | httpx.AsyncClient | None = None
+ client_verify: str | Literal[True] = True
@dataclass(frozen=True)
@@ -120,6 +121,8 @@ class _ResolvedBootstrap:
workspace: str | None
default_headers: dict[str, str]
token_provider: _AccessTokenProvider | None # None for non-OAuth users
+ client_verify: str | Literal[True]
+ certificate_authority: str | None = None
@dataclass(frozen=True)
@@ -127,7 +130,7 @@ class _ProviderCacheKey:
"""Composite key for the provider cache.
Two clients share the same provider iff they read from the same config
- file, same context, and the OIDC settings (endpoint, client, scope) match.
+ file, same context, OIDC settings (endpoint, client, scope), and CA match.
"""
config_path: Path
@@ -135,9 +138,10 @@ class _ProviderCacheKey:
token_endpoint: str
client_id: str
refresh_scope: str | None
+ certificate_authority: str | None
-# Process-wide cache: (config_path, context) → shared OIDCTokenProvider.
+# Process-wide cache: (config_path, context, OIDC settings, CA) → shared OIDCTokenProvider.
# This avoids redundant refresh-token grants when the user creates multiple
# NeMoPlatform() instances pointing at the same context.
_TOKEN_PROVIDER_CACHE: dict[_ProviderCacheKey, OIDCTokenProvider] = {}
@@ -157,7 +161,7 @@ class _ProviderCacheKey:
)
-def _discover_oidc_client_settings(base_url: str) -> NMPOIDCConfig:
+def _discover_oidc_client_settings(base_url: str, certificate_authority: str | None = None) -> NMPOIDCConfig:
"""Fetch OIDC config from the NeMo Platform cluster's discovery endpoint.
Returns a safe fallback (auth_enabled=False) if the cluster is
@@ -165,7 +169,7 @@ def _discover_oidc_client_settings(base_url: str) -> NMPOIDCConfig:
clusters work without errors during client construction.
"""
try:
- return discover_nmp_config(base_url)
+ return discover_nmp_config(base_url, certificate_authority=certificate_authority)
except Exception:
logger.debug("Could not discover OIDC settings from %s", base_url, exc_info=True)
return _OIDC_DISCOVERY_FALLBACK
@@ -177,9 +181,14 @@ def _workload_identity_token_file_from_env() -> Path | None:
return Path(token_file) if token_file else None
-def _create_workload_exchange_provider(base_url: str, subject_token_file: Path) -> WorkloadTokenExchangeProvider:
+def _create_workload_exchange_provider(
+ base_url: str,
+ subject_token_file: Path,
+ *,
+ certificate_authority: str | None = None,
+) -> WorkloadTokenExchangeProvider:
"""Create a workload identity token exchange provider from NeMo auth discovery metadata."""
- oidc_config = _discover_oidc_client_settings(base_url)
+ oidc_config = _discover_oidc_client_settings(base_url, certificate_authority=certificate_authority)
if not oidc_config.workload_token_exchange_enabled:
raise RuntimeError(
f"{WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR} is set but workload token exchange is not enabled by auth discovery"
@@ -202,6 +211,7 @@ def _create_workload_exchange_provider(base_url: str, subject_token_file: Path)
subject_token_file=subject_token_file,
audience=oidc_config.workload_audience,
scope=oidc_config.workload_scope,
+ certificate_authority=certificate_authority,
refresh_margin_seconds=_TOKEN_REFRESH_MARGIN_SECONDS,
)
@@ -209,9 +219,16 @@ def _create_workload_exchange_provider(base_url: str, subject_token_file: Path)
class _LazyWorkloadTokenExchangeProvider:
"""Create the workload exchange provider on the first token request."""
- def __init__(self, *, base_url: str, subject_token_file: Path) -> None:
+ def __init__(
+ self,
+ *,
+ base_url: str,
+ subject_token_file: Path,
+ certificate_authority: str | None = None,
+ ) -> None:
self._base_url = base_url
self._subject_token_file = subject_token_file
+ self._certificate_authority = certificate_authority
self._provider: WorkloadTokenExchangeProvider | None = None
self._lock = threading.Lock()
@@ -222,7 +239,11 @@ def _get_provider(self) -> WorkloadTokenExchangeProvider:
with self._lock:
provider = self._provider
if provider is None:
- provider = _create_workload_exchange_provider(self._base_url, self._subject_token_file)
+ provider = _create_workload_exchange_provider(
+ self._base_url,
+ self._subject_token_file,
+ certificate_authority=self._certificate_authority,
+ )
self._provider = provider
return provider
@@ -506,6 +527,8 @@ def _resolve_bootstrap(
)
base_url = str(resolved.cluster.base_url)
+ certificate_authority = resolved.cluster.certificate_authority
+ client_verify = client_verify_from_env(certificate_authority)
headers: dict[str, str] = dict(extra_headers) if extra_headers else {}
workload_identity_token_file = _workload_identity_token_file_from_env()
@@ -513,8 +536,9 @@ def _resolve_bootstrap(
provider = _LazyWorkloadTokenExchangeProvider(
base_url=base_url,
subject_token_file=workload_identity_token_file,
+ certificate_authority=certificate_authority,
)
- return _ResolvedBootstrap(base_url, resolved.workspace, headers, provider)
+ return _ResolvedBootstrap(base_url, resolved.workspace, headers, provider, client_verify, certificate_authority)
# --- Non-OAuth path (no auth) ---
if not isinstance(resolved.user, OAuthUser):
@@ -522,18 +546,18 @@ def _resolve_bootstrap(
user_headers = user_config.get("default_headers", {})
if isinstance(user_headers, dict):
headers.update(user_headers)
- return _ResolvedBootstrap(base_url, resolved.workspace, headers, None)
+ return _ResolvedBootstrap(base_url, resolved.workspace, headers, None, client_verify, certificate_authority)
# --- OAuth path: set up transparent token refresh ---
try:
- oidc_config = discover_nmp_config(base_url)
+ oidc_config = discover_nmp_config(base_url, certificate_authority=certificate_authority)
if not oidc_config.auth_enabled and access_token is None:
# Discovery succeeded and confirmed the cluster has no OIDC. The
# stored OAuthUser token can't be refreshed here (no endpoint), so
# fall back to no-auth. This guard only fires on a successful
# discovery response — not on discovery failures, where the stored
# token may still be valid and should be used as-is.
- return _ResolvedBootstrap(base_url, resolved.workspace, headers, None)
+ return _ResolvedBootstrap(base_url, resolved.workspace, headers, None, client_verify, certificate_authority)
except Exception:
logger.debug("Could not discover OIDC settings from %s", base_url, exc_info=True)
oidc_config = _OIDC_DISCOVERY_FALLBACK
@@ -560,6 +584,7 @@ def _resolve_bootstrap(
token_endpoint=token_endpoint,
client_id=client_id,
refresh_scope=refresh_scope,
+ certificate_authority=certificate_authority,
)
on_refreshed = _make_config_persister(resolved.context_name, resolved_config_path)
load_tokens = _make_config_token_loader(resolved.context_name, resolved_config_path)
@@ -573,6 +598,7 @@ def _resolve_bootstrap(
tokens=tokens,
refresh_margin_seconds=_TOKEN_REFRESH_MARGIN_SECONDS,
refresh_scope=refresh_scope,
+ certificate_authority=certificate_authority,
load_tokens=load_tokens,
refresh_lock=refresh_lock,
on_tokens_refreshed=on_refreshed,
@@ -586,9 +612,10 @@ def _resolve_bootstrap(
tokens=tokens,
refresh_margin_seconds=_TOKEN_REFRESH_MARGIN_SECONDS,
refresh_scope=refresh_scope,
+ certificate_authority=certificate_authority,
)
- return _ResolvedBootstrap(base_url, resolved.workspace, headers, provider)
+ return _ResolvedBootstrap(base_url, resolved.workspace, headers, provider, client_verify, certificate_authority)
# ---------------------------------------------------------------------------
@@ -623,6 +650,7 @@ def build_client_init_kwargs(
base_url=bootstrap.base_url,
workspace=bootstrap.workspace,
default_headers=bootstrap.default_headers or None,
+ client_verify=bootstrap.client_verify,
)
# Seed the default headers with a current token so that SDK internals
@@ -634,13 +662,14 @@ def build_client_init_kwargs(
http_client = DefaultHttpxClient(
event_hooks={"request": [hook], "response": []},
follow_redirects=True,
- verify=client_verify_from_env(),
+ verify=bootstrap.client_verify,
)
return ClientInitConfig(
base_url=bootstrap.base_url,
workspace=bootstrap.workspace,
default_headers=headers or None,
http_client=http_client,
+ client_verify=bootstrap.client_verify,
)
@@ -670,6 +699,7 @@ def build_async_client_init_kwargs(
base_url=bootstrap.base_url,
workspace=bootstrap.workspace,
default_headers=bootstrap.default_headers or None,
+ client_verify=bootstrap.client_verify,
)
headers = _headers_with_seeded_auth(bootstrap.default_headers, bootstrap.token_provider)
@@ -677,13 +707,14 @@ def build_async_client_init_kwargs(
http_client = DefaultAsyncHttpxClient(
event_hooks={"request": [hook], "response": []},
follow_redirects=True,
- verify=client_verify_from_env(),
+ verify=bootstrap.client_verify,
)
return ClientInitConfig(
base_url=bootstrap.base_url,
workspace=bootstrap.workspace,
default_headers=headers or None,
http_client=http_client,
+ client_verify=bootstrap.client_verify,
)
@@ -712,6 +743,8 @@ def create_client(
http_client = client_init_kwargs.http_client
if http_client is not None and not isinstance(http_client, httpx.Client):
raise TypeError("build_client_init_kwargs returned a non-sync HTTP client")
+ if http_client is None and client_init_kwargs.client_verify is not True:
+ http_client = DefaultHttpxClient(verify=client_init_kwargs.client_verify)
return NeMoPlatform(
config_path=config_path,
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/client/tls.py b/packages/nemo_platform_ext/src/nemo_platform_ext/client/tls.py
index b2bc998d2e..dba973f5ea 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/client/tls.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/client/tls.py
@@ -6,11 +6,51 @@
from __future__ import annotations
import os
+from collections.abc import Mapping, Sequence
+from typing import Literal, TypedDict
NMP_CLIENT_SSL_CERT_FILE_ENVVAR = "NMP_CLIENT_SSL_CERT_FILE"
-def client_verify_from_env() -> str | bool:
- """Return the httpx verify setting for NeMo Platform client requests."""
- cert_file = os.environ.get(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "").strip()
- return cert_file or True
+class HttpxTLSConfig(TypedDict, total=False):
+ """TLS kwargs passed to HTTPX client and request calls."""
+
+ verify: str
+
+
+def client_certificate_authority_from_env(
+ certificate_authority: str | None = None,
+ env: Mapping[str, str] | None = None,
+ *,
+ cert_file_envvars: Sequence[str] = (NMP_CLIENT_SSL_CERT_FILE_ENVVAR,),
+) -> str | None:
+ """Return the configured CA bundle path, if one is configured."""
+ for envvar in cert_file_envvars:
+ cert_file = (env[envvar] if env is not None and envvar in env else os.environ.get(envvar, "")).strip()
+ if cert_file:
+ return cert_file
+ return certificate_authority or None
+
+
+def httpx_tls_config_from_env(
+ certificate_authority: str | None = None,
+ env: Mapping[str, str] | None = None,
+ *,
+ cert_file_envvars: Sequence[str] = (NMP_CLIENT_SSL_CERT_FILE_ENVVAR,),
+) -> HttpxTLSConfig:
+ """Return HTTPX TLS kwargs for NeMo Platform client requests."""
+ ca_bundle = client_certificate_authority_from_env(
+ certificate_authority,
+ env=env,
+ cert_file_envvars=cert_file_envvars,
+ )
+ return {"verify": ca_bundle} if ca_bundle is not None else {}
+
+
+def client_verify_from_env(certificate_authority: str | None = None) -> str | Literal[True]:
+ """Return the httpx verify setting for NeMo Platform client requests.
+
+ The environment variable remains an explicit runtime override. A saved
+ cluster ``certificate_authority`` is used when no override is set.
+ """
+ return client_certificate_authority_from_env(certificate_authority) or True
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py b/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py
index 8addf10df3..b839fd4daa 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py
@@ -90,6 +90,10 @@ class Config(BaseModel):
current_context: str | None = Field(default=None, description="Override current context (env: NMP_CURRENT_CONTEXT)")
base_url: str | None = Field(default=None, description="Base URL for the API (env: NMP_BASE_URL)")
+ certificate_authority: str | None = Field(
+ default=None,
+ description="Path to a PEM certificate authority bundle for TLS verification",
+ )
access_token: SecretStr | None = Field(
default=None,
description="Access token for authentication (env: NMP_ACCESS_TOKEN)",
@@ -425,6 +429,63 @@ def set_current_context(self, context_name: str) -> None:
# Save to disk
self.save()
+ @classmethod
+ def delete_context(
+ cls,
+ context_name: str,
+ config_path: Path | None = None,
+ *,
+ prune_orphans: bool = False,
+ ) -> ConfigDeleteContextResult[Self]:
+ """Delete a context definition from the config file.
+
+ This follows kubectl's convention: deleting a context only removes the
+ context reference. Referenced cluster and user records are left intact
+ unless ``prune_orphans`` is explicitly requested.
+ """
+ path = config_path or cls.get_default_config_path()
+ config = cls.load(config_path=path)
+ config_file = config._config_file
+
+ context_index = next(
+ (index for index, context in enumerate(config_file.contexts) if context.name == context_name),
+ None,
+ )
+ if context_index is None:
+ available = (
+ ", ".join(context.name for context in config_file.contexts) if config_file.contexts else "(none)"
+ )
+ raise ValueError(f"Context '{context_name}' not found. Available contexts: {available}")
+
+ del config_file.contexts[context_index]
+
+ current_context_cleared = config_file.current_context == context_name
+ if current_context_cleared:
+ config_file.current_context = None
+
+ pruned_clusters: list[str] = []
+ pruned_users: list[str] = []
+ if prune_orphans:
+ referenced_clusters = {context.cluster for context in config_file.contexts}
+ referenced_users = {context.user for context in config_file.contexts}
+
+ pruned_clusters = [
+ cluster.name for cluster in config_file.clusters if cluster.name not in referenced_clusters
+ ]
+ pruned_users = [user.name for user in config_file.users if user.name not in referenced_users]
+
+ config_file.clusters = [cluster for cluster in config_file.clusters if cluster.name in referenced_clusters]
+ config_file.users = [user for user in config_file.users if user.name in referenced_users]
+
+ config.save()
+ return ConfigDeleteContextResult(
+ config=config,
+ context_name=context_name,
+ current_context_cleared=current_context_cleared,
+ pruned_clusters=pruned_clusters,
+ pruned_users=pruned_users,
+ )
+
def reload(self) -> None:
"""Reload configuration from file (preserving runtime parameters)."""
if self._config_path:
@@ -439,6 +500,7 @@ def reload(self) -> None:
self.current_context = new_config.current_context
self.base_url = new_config.base_url
+ self.certificate_authority = new_config.certificate_authority
self.access_token = new_config.access_token
self.workspace = new_config.workspace
@@ -492,6 +554,7 @@ def resolve(self) -> Context:
if cluster is None:
raise ValueError(f"Cluster '{context.cluster}' referenced by context '{context_name}' not found")
+ cluster = cluster.model_copy(deep=True)
# Resolve the user reference (string) into AuthConfig
user = None
@@ -509,6 +572,8 @@ def resolve(self) -> Context:
# Apply runtime overrides
if self.base_url is not None:
cluster.base_url = HttpUrl(self.base_url)
+ if self.certificate_authority is not None:
+ cluster.certificate_authority = self.certificate_authority
if self.access_token is not None:
user = OAuthUser(
name=user.name,
@@ -570,6 +635,8 @@ def _create_default_config(self) -> Context:
# Build params from runtime overrides
params: ConfigParams = {"base_url": base_url}
+ if self.certificate_authority is not None:
+ params["certificate_authority"] = self.certificate_authority
if self.access_token:
params["access_token"] = self.access_token.get_secret_value()
if self.workspace:
@@ -620,6 +687,17 @@ class ConfigWriteResult(Generic[_T]):
created: bool
+@dataclass(frozen=True)
+class ConfigDeleteContextResult(Generic[_T]):
+ """Result metadata for a context deletion."""
+
+ config: _T
+ context_name: str
+ current_context_cleared: bool
+ pruned_clusters: list[str]
+ pruned_users: list[str]
+
+
def get_context(
config_path: Path | None = None,
overrides: ConfigParams | None = None,
diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/config/models.py b/packages/nemo_platform_ext/src/nemo_platform_ext/config/models.py
index 689cebc8ae..b3caa08fd7 100644
--- a/packages/nemo_platform_ext/src/nemo_platform_ext/config/models.py
+++ b/packages/nemo_platform_ext/src/nemo_platform_ext/config/models.py
@@ -104,6 +104,10 @@ class Cluster(BaseModel):
name: str = Field(..., min_length=1, description="Unique cluster name")
base_url: HttpUrl = Field(..., description="Base URL for the cluster")
+ certificate_authority: str | None = Field(
+ default=None,
+ description="Path to a PEM certificate authority bundle for TLS verification",
+ )
metadata: dict[str, Any] = Field(default_factory=dict, description="Additional cluster metadata")
@field_validator("name")
@@ -169,6 +173,8 @@ class ConfigParams(TypedDict, total=False):
base_url: str
+ certificate_authority: str | None
+
# OAuth fields (for OAuthUser)
access_token: str | None
refresh_token: str | None
@@ -260,6 +266,8 @@ def ensure_context(
self.clusters.append(cluster)
elif "base_url" in params:
cluster.base_url = HttpUrl(params["base_url"])
+ if "certificate_authority" in params:
+ cluster.certificate_authority = params["certificate_authority"]
# Find existing or create user
user: User = next((u for u in self.users if u.name == user_name), None) # type: ignore[assignment]
diff --git a/packages/nemo_platform_ext/tests/auth/test_device_flow.py b/packages/nemo_platform_ext/tests/auth/test_device_flow.py
index 4672af4d7d..e5a1e0669e 100644
--- a/packages/nemo_platform_ext/tests/auth/test_device_flow.py
+++ b/packages/nemo_platform_ext/tests/auth/test_device_flow.py
@@ -137,6 +137,37 @@ async def test_start_device_authorization_success(self, device_flow):
},
timeout=30.0,
)
+ mock_client_class.assert_called_once_with()
+
+ @pytest.mark.asyncio
+ async def test_start_device_authorization_uses_context_certificate_authority(self, monkeypatch):
+ """Test device authorization uses an explicit context CA bundle."""
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+ flow = DeviceFlow(
+ device_authorization_endpoint="https://sso.example.com/device/code",
+ token_endpoint="https://sso.example.com/oauth/token",
+ client_id="test-client",
+ certificate_authority="/tmp/context-ca.pem",
+ )
+
+ with patch("httpx.AsyncClient") as mock_client_class:
+ mock_client = AsyncMock()
+ mock_response = MagicMock()
+ mock_response.raise_for_status = MagicMock()
+ mock_response.json.return_value = {
+ "device_code": "device_code_123",
+ "user_code": "ABC-123",
+ "verification_uri": "https://sso.example.com/device",
+ "expires_in": 1800,
+ }
+ mock_client.post.return_value = mock_response
+ mock_client.__aenter__.return_value = mock_client
+ mock_client.__aexit__.return_value = None
+ mock_client_class.return_value = mock_client
+
+ await flow.start_device_authorization()
+
+ mock_client_class.assert_called_once_with(verify="/tmp/context-ca.pem")
@pytest.mark.asyncio
async def test_start_device_authorization_uses_nemo_scoped_ca_bundle(self, device_flow, monkeypatch):
@@ -371,8 +402,27 @@ async def test_refresh_access_token_success(self):
"client-id",
"refresh-token",
scope="openid profile",
+ certificate_authority=None,
+ )
+
+ @pytest.mark.asyncio
+ async def test_refresh_access_token_uses_context_certificate_authority(self):
+ with patch("nemo_platform_ext.auth.device_flow.refresh_token_grant") as mock_refresh:
+ mock_refresh.return_value = {
+ "access_token": "new_access",
+ "token_type": "Bearer",
+ "expires_in": 3600,
+ }
+
+ await refresh_access_token(
+ token_endpoint="https://idp/token",
+ client_id="client-id",
+ refresh_token="refresh-token",
+ certificate_authority="/tmp/context-ca.pem",
)
+ assert mock_refresh.call_args.kwargs["certificate_authority"] == "/tmp/context-ca.pem"
+
@pytest.mark.asyncio
async def test_refresh_access_token_failure(self):
with patch("nemo_platform_ext.auth.device_flow.refresh_token_grant") as mock_refresh:
@@ -427,6 +477,7 @@ def test_authenticate_with_password_grant_success(self):
},
timeout=30.0,
)
+ mock_client_class.assert_called_once_with()
def test_authenticate_with_password_grant_uses_nemo_scoped_ca_bundle(self, monkeypatch):
monkeypatch.setenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "/tmp/nemo-ca.pem")
@@ -454,6 +505,33 @@ def test_authenticate_with_password_grant_uses_nemo_scoped_ca_bundle(self, monke
mock_client_class.assert_called_once_with(verify="/tmp/nemo-ca.pem")
+ def test_authenticate_with_password_grant_uses_context_certificate_authority(self, monkeypatch):
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ with patch("httpx.Client") as mock_client_class:
+ mock_client = MagicMock()
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "access_token": "access_123",
+ "token_type": "Bearer",
+ "expires_in": 3600,
+ }
+ mock_client.post.return_value = mock_response
+ mock_client.__enter__.return_value = mock_client
+ mock_client.__exit__.return_value = None
+ mock_client_class.return_value = mock_client
+
+ authenticate_with_password_grant(
+ token_endpoint="https://idp/token",
+ client_id="client-id",
+ username="user",
+ password="secret",
+ certificate_authority="/tmp/context-ca.pem",
+ )
+
+ mock_client_class.assert_called_once_with(verify="/tmp/context-ca.pem")
+
def test_authenticate_with_password_grant_failure(self):
with patch("httpx.Client") as mock_client_class:
mock_client = MagicMock()
diff --git a/packages/nemo_platform_ext/tests/auth/test_token_provider.py b/packages/nemo_platform_ext/tests/auth/test_token_provider.py
index 785d14cdf3..abaa96a849 100644
--- a/packages/nemo_platform_ext/tests/auth/test_token_provider.py
+++ b/packages/nemo_platform_ext/tests/auth/test_token_provider.py
@@ -124,6 +124,25 @@ def test_refresh_token_grant_uses_nemo_scoped_ca_bundle(self, mock_post, monkeyp
assert result == {"access_token": "new_access"}
assert mock_post.call_args.kwargs["verify"] == "/tmp/nemo-ca.pem"
+ @patch("nemo_platform_ext.auth.token_provider.httpx.post")
+ def test_refresh_token_grant_uses_context_certificate_authority(self, mock_post, tmp_path, monkeypatch):
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"access_token": "new_access"}
+ mock_post.return_value = mock_response
+ context_ca = str(tmp_path / "context-ca.pem")
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ result = refresh_token_grant(
+ token_endpoint="https://idp/token",
+ client_id="client",
+ refresh_token="refresh_abc",
+ certificate_authority=context_ca,
+ )
+
+ assert result == {"access_token": "new_access"}
+ assert mock_post.call_args.kwargs["verify"] == context_ca
+
@patch("nemo_platform_ext.auth.token_provider.httpx.post")
def test_get_access_token_refreshes_when_expired(self, mock_post):
old_token = _make_jwt({"exp": int(time.time()) - 100})
diff --git a/packages/nemo_platform_ext/tests/auth/test_utils.py b/packages/nemo_platform_ext/tests/auth/test_utils.py
index 6c66577652..6c8180b030 100644
--- a/packages/nemo_platform_ext/tests/auth/test_utils.py
+++ b/packages/nemo_platform_ext/tests/auth/test_utils.py
@@ -178,6 +178,23 @@ def test_uses_nemo_scoped_ca_bundle(self, mock_get, monkeypatch):
verify="/tmp/nemo-ca.pem",
)
+ @patch("nemo_platform_ext.auth.helpers.httpx.get")
+ def test_uses_context_certificate_authority(self, mock_get, tmp_path, monkeypatch):
+ response = MagicMock()
+ response.json.return_value = {"auth_enabled": False}
+ mock_get.return_value = response
+ context_ca = str(tmp_path / "context-ca.pem")
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ result = discover_nmp_config("https://nemo.example.com", certificate_authority=context_ca)
+
+ assert result.auth_enabled is False
+ mock_get.assert_called_once_with(
+ "https://nemo.example.com/apis/auth/discovery",
+ timeout=10.0,
+ verify=context_ca,
+ )
+
class TestBuildEffectiveScope:
def test_no_prefix_returns_unchanged(self):
diff --git a/packages/nemo_platform_ext/tests/auth/test_workload_exchange.py b/packages/nemo_platform_ext/tests/auth/test_workload_exchange.py
index f64f226776..8d63685283 100644
--- a/packages/nemo_platform_ext/tests/auth/test_workload_exchange.py
+++ b/packages/nemo_platform_ext/tests/auth/test_workload_exchange.py
@@ -131,6 +131,27 @@ def test_token_exchange_grant_uses_nemo_scoped_ca_bundle(mock_post, monkeypatch)
assert mock_post.call_args.kwargs["verify"] == "/tmp/nemo-ca.pem"
+@patch("nemo_platform_ext.auth.workload_exchange.httpx.post")
+def test_token_exchange_grant_uses_context_certificate_authority(mock_post, tmp_path, monkeypatch):
+ access_token = _make_jwt({"exp": int(time.time()) + 3600})
+ response = MagicMock()
+ response.status_code = 200
+ response.json.return_value = {"access_token": access_token}
+ mock_post.return_value = response
+ context_ca = str(tmp_path / "context-ca.pem")
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ result = token_exchange_grant(
+ token_endpoint="https://idp.example.com/token",
+ client_id="nemo-platform-workload",
+ subject_token="subject-token",
+ certificate_authority=context_ca,
+ )
+
+ assert result["access_token"] == access_token
+ assert mock_post.call_args.kwargs["verify"] == context_ca
+
+
@patch("nemo_platform_ext.auth.workload_exchange.httpx.post")
def test_token_exchange_grant_surfaces_idp_error(mock_post):
response = MagicMock()
@@ -242,6 +263,7 @@ def test_provider_rejects_expired_exchange_response_and_retries_with_current_sub
subject_token_file=subject_token_file,
audience="nemo-platform",
scope="openid email groups",
+ certificate_authority="/tmp/context-ca.pem",
refresh_margin_seconds=0,
)
@@ -256,3 +278,4 @@ def test_provider_rejects_expired_exchange_response_and_retries_with_current_sub
assert mock_exchange.call_args_list[1].kwargs["subject_token"] == "subject-token-two"
assert mock_exchange.call_args_list[1].kwargs["audience"] == "nemo-platform"
assert mock_exchange.call_args_list[1].kwargs["scope"] == "openid email groups"
+ assert mock_exchange.call_args_list[1].kwargs["certificate_authority"] == "/tmp/context-ca.pem"
diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_auth.py b/packages/nemo_platform_ext/tests/cli/commands/test_auth.py
index d2b108dcb1..d6c77f4b33 100644
--- a/packages/nemo_platform_ext/tests/cli/commands/test_auth.py
+++ b/packages/nemo_platform_ext/tests/cli/commands/test_auth.py
@@ -48,19 +48,25 @@ def _mock_oidc_config() -> SimpleNamespace:
)
-def _discover_auth_enabled(url: str, timeout: float = 10.0) -> SimpleNamespace:
+def _discover_auth_enabled(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+) -> SimpleNamespace:
return SimpleNamespace(auth_enabled=True)
-def _discover_auth_disabled(url: str, timeout: float = 10.0) -> SimpleNamespace:
+def _discover_auth_disabled(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+) -> SimpleNamespace:
return SimpleNamespace(auth_enabled=False)
-def _discover_oidc_config(url: str, timeout: float = 10.0) -> SimpleNamespace:
+def _discover_oidc_config(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+) -> SimpleNamespace:
return _mock_oidc_config()
-def _discover_no_oidc(url: str, timeout: float = 10.0) -> SimpleNamespace:
+def _discover_no_oidc(url: str, timeout: float = 10.0, *, certificate_authority: str | None = None) -> SimpleNamespace:
return SimpleNamespace(
auth_enabled=True,
issuer=None,
@@ -300,7 +306,9 @@ def __init__(self, *args, **kwargs):
def force_refresh(self) -> None:
return None
- def discover_refresh_config(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def discover_refresh_config(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
return SimpleNamespace(
client_id="test-client-id",
token_endpoint="https://idp.example.com/token",
@@ -919,6 +927,52 @@ def password_grant(**kwargs) -> SimpleNamespace:
assert foo_user["refresh_token"] == "foo-refresh-token"
+def test_auth_login_uses_context_certificate_authority(
+ oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ with open(oauth_config_file) as f:
+ config_data = yaml.safe_load(f)
+ foo_cluster = next(cluster for cluster in config_data["clusters"] if cluster["name"] == "foo")
+ foo_cluster["certificate_authority"] = "/tmp/foo-ca.pem"
+ with open(oauth_config_file, "w") as f:
+ yaml.safe_dump(config_data, f)
+
+ discover_calls: list[tuple[str, str | None]] = []
+ password_grant_calls: list[dict] = []
+
+ def discover_config(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
+ discover_calls.append((url, certificate_authority))
+ return _mock_oidc_config()
+
+ def password_grant(**kwargs) -> SimpleNamespace:
+ password_grant_calls.append(kwargs)
+ return SimpleNamespace(token_for_nmp="foo-access-token", refresh_token="foo-refresh-token")
+
+ monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", discover_config)
+ monkeypatch.setattr("nemo_platform_ext.auth.device_flow.authenticate_with_password_grant", password_grant)
+ monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.decode_jwt_claims", _decode_jwt_noop)
+
+ result = runner.invoke(
+ app,
+ [
+ "--context",
+ "foo",
+ "auth",
+ "login",
+ "--username",
+ "user",
+ "--password",
+ "secret",
+ ],
+ )
+
+ assert_exit_code(result, 0)
+ assert discover_calls == [("https://foo.example.com", "/tmp/foo-ca.pem")]
+ assert password_grant_calls[0]["certificate_authority"] == "/tmp/foo-ca.pem"
+
+
def test_auth_login_with_base_url_creates_selected_context(oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch):
def password_grant(**kwargs) -> SimpleNamespace:
return SimpleNamespace(token_for_nmp="dev-access-token", refresh_token="dev-refresh-token")
@@ -1008,7 +1062,9 @@ def test_auth_login_unsigned_options_require_unsigned_token() -> None:
def test_auth_login_unsigned_token_fails_when_oidc_enabled(
oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
- def discover_full_oidc(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def discover_full_oidc(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
return SimpleNamespace(
auth_enabled=True,
issuer="https://idp.example.com",
@@ -1039,7 +1095,9 @@ def discover_full_oidc(url: str, timeout: float = 10.0) -> SimpleNamespace:
def test_auth_login_unsigned_token_allows_partial_oidc_config(
oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
- def discover_partial_oidc(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def discover_partial_oidc(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
return SimpleNamespace(
auth_enabled=True,
issuer="https://idp.example.com",
@@ -1117,7 +1175,7 @@ class IsAuthDisabledCase:
ids=lambda c: c.id,
)
def test_is_auth_disabled(monkeypatch: pytest.MonkeyPatch, case: IsAuthDisabledCase) -> None:
- def mock_discover(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def mock_discover(url: str, timeout: float = 10.0, *, certificate_authority: str | None = None) -> SimpleNamespace:
return SimpleNamespace(auth_enabled=case.auth_enabled)
monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", mock_discover)
@@ -1131,7 +1189,9 @@ def test_is_auth_disabled_raises_when_discovery_fails(monkeypatch: pytest.Monkey
from nemo_platform_ext.auth.helpers import AuthError
from nemo_platform_ext.cli.commands.auth import is_auth_disabled
- def raise_connect_error(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def raise_connect_error(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
raise httpx.ConnectError("Connection refused")
monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", raise_connect_error)
@@ -1169,7 +1229,9 @@ def test_auth_status_when_cluster_unreachable_shows_local_state(
) -> None:
import httpx
- def raise_connect_error(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def raise_connect_error(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
raise httpx.ConnectError("Connection refused")
monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", raise_connect_error)
@@ -1211,7 +1273,9 @@ def test_auth_logout_when_cluster_unreachable_clears_local_credentials(
) -> None:
import httpx
- def raise_connect_error(url: str, timeout: float = 10.0) -> SimpleNamespace:
+ def raise_connect_error(
+ url: str, timeout: float = 10.0, *, certificate_authority: str | None = None
+ ) -> SimpleNamespace:
raise httpx.ConnectError("Connection refused")
monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", raise_connect_error)
diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_auth_password_grant.py b/packages/nemo_platform_ext/tests/cli/commands/test_auth_password_grant.py
index 58e5c59907..3476763e24 100644
--- a/packages/nemo_platform_ext/tests/cli/commands/test_auth_password_grant.py
+++ b/packages/nemo_platform_ext/tests/cli/commands/test_auth_password_grant.py
@@ -27,7 +27,10 @@ def _mock_oidc_config() -> SimpleNamespace:
def test_login_password_grant_with_flags(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("NMP_BASE_URL", "https://cluster.example.com")
- monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", lambda *_: _mock_oidc_config())
+ monkeypatch.setattr(
+ "nemo_platform_ext.cli.commands.auth.discover_nmp_config",
+ lambda *_args, **_kwargs: _mock_oidc_config(),
+ )
monkeypatch.setattr(
"nemo_platform_ext.auth.device_flow.authenticate_with_password_grant",
lambda **_: SimpleNamespace(token_for_nmp="access-token", refresh_token="refresh-token"),
@@ -62,7 +65,10 @@ def test_login_password_grant_with_env(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("NMP_BASE_URL", "https://cluster.example.com")
monkeypatch.setenv("NMP_OIDC_USERNAME", "env-user")
monkeypatch.setenv("NMP_OIDC_PASSWORD", "env-password")
- monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", lambda *_: _mock_oidc_config())
+ monkeypatch.setattr(
+ "nemo_platform_ext.cli.commands.auth.discover_nmp_config",
+ lambda *_args, **_kwargs: _mock_oidc_config(),
+ )
monkeypatch.setattr(
"nemo_platform_ext.auth.device_flow.authenticate_with_password_grant",
lambda **_: SimpleNamespace(token_for_nmp="access-token", refresh_token=None),
diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_config.py b/packages/nemo_platform_ext/tests/cli/commands/test_config.py
index be2efa35fd..7cb5cf0cf4 100644
--- a/packages/nemo_platform_ext/tests/cli/commands/test_config.py
+++ b/packages/nemo_platform_ext/tests/cli/commands/test_config.py
@@ -143,6 +143,57 @@ def test_use_nonexistent_context(config_file: Path):
assert "not found" in result.output
+def test_delete_context_removes_only_context_by_default(config_file: Path):
+ result = runner.invoke(app, "config delete-context production")
+ assert_exit_code(result, 0)
+ assert result.output.strip() == "Deleted context 'production'"
+
+ with open(config_file) as f:
+ data = yaml.safe_load(f)
+
+ assert {context["name"] for context in data["contexts"]} == {"local"}
+ assert {cluster["name"] for cluster in data["clusters"]} == {"local", "production"}
+ assert {user["name"] for user in data["users"]} == {"local", "production"}
+ assert data["current_context"] == "local"
+
+
+def test_delete_current_context_clears_current_context(config_file: Path):
+ result = runner.invoke(app, "config delete-context local")
+ assert_exit_code(result, 0)
+ assert result.output.strip() == "Deleted context 'local'\nCurrent context cleared"
+
+ with open(config_file) as f:
+ data = yaml.safe_load(f)
+
+ assert {context["name"] for context in data["contexts"]} == {"production"}
+ assert data.get("current_context") is None
+
+
+def test_delete_context_prune_orphans_removes_unreferenced_records(config_file: Path):
+ result = runner.invoke(app, "config delete-context production --prune-orphans")
+ assert_exit_code(result, 0)
+ assert "Deleted context 'production'" in result.output
+ assert "Pruned clusters: production" in result.output
+ assert "Pruned users: production" in result.output
+
+ with open(config_file) as f:
+ data = yaml.safe_load(f)
+
+ assert {context["name"] for context in data["contexts"]} == {"local"}
+ assert {cluster["name"] for cluster in data["clusters"]} == {"local"}
+ assert {user["name"] for user in data["users"]} == {"local"}
+
+
+def test_delete_context_rejects_unknown_context(config_file: Path):
+ _ = config_file
+ result = runner.invoke(app, "config delete-context missing")
+ assert_exit_code(result, 1)
+ assert "Context 'missing' not found" in result.output
+ assert "Available contexts:" in result.output
+ assert "local" in result.output
+ assert "production" in result.output
+
+
def test_set_workspace(config_file: Path):
result = runner.invoke(app, "config set --workspace new-workspace")
assert_exit_code(result, 0)
@@ -265,6 +316,35 @@ def test_set_base_url_does_not_affect_other_contexts(config_file: Path):
assert a_cluster["base_url"] == "https://a-updated.example.com/"
+def test_set_certificate_authority_updates_selected_context_cluster(config_file: Path):
+ result = runner.invoke(app, "config set --context production --certificate-authority /tmp/nemo-ca.pem")
+ assert_exit_code(result, 0)
+
+ with open(config_file) as f:
+ data = yaml.safe_load(f)
+
+ local_cluster = next(c for c in data["clusters"] if c["name"] == "local")
+ production_cluster = next(c for c in data["clusters"] if c["name"] == "production")
+ assert "certificate_authority" not in local_cluster
+ assert production_cluster["certificate_authority"] == "/tmp/nemo-ca.pem"
+
+
+def test_set_new_context_can_set_certificate_authority(config_file: Path):
+ result = runner.invoke(
+ app,
+ "config set --context local-tls --base-url https://localhost:8443 --certificate-authority /tmp/local-ca.pem",
+ )
+ assert_exit_code(result, 0)
+
+ with open(config_file) as f:
+ data = yaml.safe_load(f)
+
+ context = next(c for c in data["contexts"] if c["name"] == "local-tls")
+ cluster = next(c for c in data["clusters"] if c["name"] == context["cluster"])
+ assert cluster["base_url"] == "https://localhost:8443/"
+ assert cluster["certificate_authority"] == "/tmp/local-ca.pem"
+
+
def test_set_new_named_context_does_not_switch(config_file: Path):
result = runner.invoke(app, "config set --context new-ctx --base-url https://new.example.com")
assert_exit_code(result, 0)
@@ -378,7 +458,6 @@ def test_set_rejects_invalid_base_url(config_file: Path, invalid_url: str):
"command_name",
[
"delete-cluster",
- "delete-context",
"delete-user",
"get-clusters",
"get-contexts",
@@ -401,20 +480,24 @@ def test_config_help_emphasizes_core_flow():
assert "config view" in result.output
assert "config view --minify" not in result.output
assert "config use-context" in result.output
+ assert "config delete-context" in result.output
assert "Advanced:" not in result.output
view_match = re.search(r"^\s*view\s+", result.output, re.MULTILINE)
set_match = re.search(r"^\s*set\s+", result.output, re.MULTILINE)
current_context_match = re.search(r"^\s*current-context\s+", result.output, re.MULTILINE)
use_context_match = re.search(r"^\s*use-context\s+", result.output, re.MULTILINE)
+ delete_context_match = re.search(r"^\s*delete-context\s+", result.output, re.MULTILINE)
assert view_match is not None
assert set_match is not None
assert current_context_match is not None
assert use_context_match is not None
+ assert delete_context_match is not None
assert view_match.start() < set_match.start()
assert set_match.start() < current_context_match.start()
assert current_context_match.start() < use_context_match.start()
+ assert use_context_match.start() < delete_context_match.start()
def test_view_help_uses_all_contexts_option():
diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_setup.py b/packages/nemo_platform_ext/tests/cli/commands/test_setup.py
index a8feeb9e35..7ab6f1cf1e 100644
--- a/packages/nemo_platform_ext/tests/cli/commands/test_setup.py
+++ b/packages/nemo_platform_ext/tests/cli/commands/test_setup.py
@@ -197,7 +197,7 @@ def test_reachable_via_status(self):
mock_resp = MagicMock()
mock_resp.status_code = 200
with (
- patch(f"{SETUP_MOD}.client_verify_from_env", return_value="/tmp/custom-ca.pem"),
+ patch(f"{SETUP_MOD}.httpx_tls_config_from_env", return_value={"verify": "/tmp/custom-ca.pem"}),
patch(f"{SETUP_MOD}.httpx.get", return_value=mock_resp) as mock_get,
):
assert _check_platform_reachable("http://localhost:8080") is True
@@ -223,7 +223,7 @@ def _get(url, **kwargs):
raise AssertionError(f"unexpected url: {url}")
with (
- patch(f"{SETUP_MOD}.client_verify_from_env", return_value="/tmp/custom-ca.pem"),
+ patch(f"{SETUP_MOD}.httpx_tls_config_from_env", return_value={"verify": "/tmp/custom-ca.pem"}),
patch(f"{SETUP_MOD}.httpx.get", side_effect=_get),
):
assert _check_platform_reachable("https://nemo-platform-freeplay.dev.aire.nvidia.com") is True
diff --git a/packages/nemo_platform_ext/tests/client/test_client.py b/packages/nemo_platform_ext/tests/client/test_client.py
index e4b79332f9..2b0eaff146 100644
--- a/packages/nemo_platform_ext/tests/client/test_client.py
+++ b/packages/nemo_platform_ext/tests/client/test_client.py
@@ -30,7 +30,15 @@ def _make_jwt(claims: dict) -> str:
return f"{h}.{p}.{s}"
-def _write_config(tmp_path, *, user_type="oauth", token=None, refresh_token=None, api_key=None):
+def _write_config(
+ tmp_path,
+ *,
+ user_type="oauth",
+ token=None,
+ refresh_token=None,
+ api_key=None,
+ certificate_authority=None,
+):
"""Write a minimal nmp config file and return its path."""
if user_type == "oauth":
user = {
@@ -44,9 +52,13 @@ def _write_config(tmp_path, *, user_type="oauth", token=None, refresh_token=None
else:
user = {"name": "default", "type": "no-auth"}
+ cluster = {"name": "default", "base_url": "http://localhost:8080"}
+ if certificate_authority:
+ cluster["certificate_authority"] = certificate_authority
+
config = {
"current_context": "default",
- "clusters": [{"name": "default", "base_url": "http://localhost:8080"}],
+ "clusters": [cluster],
"users": [user],
"contexts": [
{
@@ -150,6 +162,57 @@ def test_oauth_uses_nemo_scoped_ca_bundle(self, _mock_discover, mock_default_htt
assert mock_default_httpx_client.call_args.kwargs["verify"] == "/tmp/nemo-ca.pem"
+ @patch("nemo_platform_ext.client.factory.DefaultHttpxClient")
+ @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
+ def test_oauth_uses_context_certificate_authority(
+ self, _mock_discover, mock_default_httpx_client, tmp_path, monkeypatch
+ ):
+ token = _make_jwt({"exp": int(time.time()) + 3600, "sub": "user1"})
+ context_ca = str(tmp_path / "context-ca.pem")
+ config_path = _write_config(
+ tmp_path,
+ token=token,
+ refresh_token="refresh_abc",
+ certificate_authority=context_ca,
+ )
+ http_client = httpx.Client()
+ mock_default_httpx_client.return_value = http_client
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ client = create_client(config_path=config_path)
+ try:
+ assert client is not None
+ finally:
+ client.close()
+
+ assert mock_default_httpx_client.call_args.kwargs["verify"] == context_ca
+ assert _mock_discover.call_args.kwargs["certificate_authority"] == context_ca
+
+ @patch("nemo_platform_ext.client.factory.DefaultHttpxClient")
+ @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
+ def test_env_ca_bundle_overrides_context_certificate_authority(
+ self, _mock_discover, mock_default_httpx_client, tmp_path, monkeypatch
+ ):
+ token = _make_jwt({"exp": int(time.time()) + 3600, "sub": "user1"})
+ config_path = _write_config(
+ tmp_path,
+ token=token,
+ refresh_token="refresh_abc",
+ certificate_authority="/tmp/context-ca.pem",
+ )
+ http_client = httpx.Client()
+ mock_default_httpx_client.return_value = http_client
+ monkeypatch.setenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "/tmp/env-ca.pem")
+
+ client = create_client(config_path=config_path)
+ try:
+ assert client is not None
+ finally:
+ client.close()
+
+ assert mock_default_httpx_client.call_args.kwargs["verify"] == "/tmp/env-ca.pem"
+ assert _mock_discover.call_args.kwargs["certificate_authority"] == "/tmp/context-ca.pem"
+
@patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
@patch("nemo_platform_ext.auth.token_provider.httpx.post")
def test_persist_refreshed_tokens_writes_to_config(self, mock_post, _mock_discover, tmp_path):
@@ -298,6 +361,39 @@ def test_exchanges_workload_identity_token_file(self, mock_exchange, _mock_disco
assert mock_exchange.call_args.kwargs["audience"] == "nemo-platform"
assert mock_exchange.call_args.kwargs["scope"] == "openid email groups"
+ @patch("nemo_platform_ext.client.factory.DefaultHttpxClient")
+ @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_WORKLOAD_NMP_CONFIG)
+ @patch("nemo_platform_ext.auth.workload_exchange.token_exchange_grant")
+ def test_workload_identity_discovery_uses_context_certificate_authority(
+ self, mock_exchange, _mock_discover, mock_default_httpx_client, tmp_path, monkeypatch
+ ):
+ subject_token_file = tmp_path / "workload-token"
+ subject_token_file.write_text("subject-token-one\n", encoding="utf-8")
+ context_ca = str(tmp_path / "context-ca.pem")
+ access_token = _make_jwt({"exp": int(time.time()) + 3600, "sub": "workload-user"})
+ mock_exchange.return_value = {"access_token": access_token, "expires_in": 300}
+ config_path = _write_config(tmp_path, user_type="no-auth", certificate_authority=context_ca)
+ monkeypatch.setenv(WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, str(subject_token_file))
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ def default_httpx_client(*args, **kwargs):
+ kwargs["verify"] = True
+ return httpx.Client(*args, **kwargs)
+
+ mock_default_httpx_client.side_effect = default_httpx_client
+
+ client = create_client(config_path=config_path)
+ try:
+ request = client._client.build_request("GET", "http://localhost:8080/test")
+ client._client._event_hooks["request"][0](request)
+ assert request.headers["Authorization"] == f"Bearer {access_token}"
+ finally:
+ client.close()
+
+ assert _mock_discover.call_args.kwargs["certificate_authority"] == context_ca
+ assert mock_exchange.call_args.kwargs["certificate_authority"] == context_ca
+ assert mock_default_httpx_client.call_args.kwargs["verify"] == context_ca
+
@pytest.mark.asyncio
@patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_WORKLOAD_NMP_CONFIG)
@patch("nemo_platform_ext.auth.workload_exchange.token_exchange_grant")
@@ -386,6 +482,28 @@ def test_creates_client_without_auth(self, tmp_path):
# No auth headers should be set
assert "Authorization" not in client._custom_headers
+ @patch("nemo_platform_ext.client.factory.DefaultHttpxClient")
+ def test_non_oauth_context_uses_context_certificate_authority(
+ self, mock_default_httpx_client, tmp_path, monkeypatch
+ ):
+ context_ca = str(tmp_path / "context-ca.pem")
+ config_path = _write_config(
+ tmp_path,
+ user_type="no-auth",
+ certificate_authority=context_ca,
+ )
+ http_client = httpx.Client()
+ mock_default_httpx_client.return_value = http_client
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ client = create_client(config_path=config_path)
+ try:
+ assert str(client.base_url).rstrip("/") == "http://localhost:8080"
+ finally:
+ client.close()
+
+ mock_default_httpx_client.assert_called_once_with(verify=context_ca)
+
class TestCreateClientTimeout:
@patch("nemo_platform_ext.client.factory.NeMoPlatform")
@@ -429,6 +547,50 @@ def test_reuses_oauth_provider_for_same_context(self, mock_provider_cls, _mock_d
assert callable(provider_kwargs["load_tokens"])
assert callable(provider_kwargs["refresh_lock"])
+ @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
+ @patch("nemo_platform_ext.client.factory.DefaultHttpxClient")
+ @patch("nemo_platform_ext.client.factory.OIDCTokenProvider")
+ def test_context_certificate_authority_participates_in_provider_cache_key(
+ self, mock_provider_cls, mock_default_httpx_client, _mock_discover, tmp_path, monkeypatch
+ ):
+ token = _make_jwt({"exp": int(time.time()) + 3600, "sub": "user1"})
+ first_ca = str(tmp_path / "first-ca.pem")
+ second_ca = str(tmp_path / "second-ca.pem")
+ config_path = _write_config(
+ tmp_path,
+ token=token,
+ refresh_token="refresh_abc",
+ certificate_authority=first_ca,
+ )
+ first_provider = MagicMock()
+ first_provider.get_access_token.return_value = token
+ first_provider.reload_tokens.return_value = False
+ second_provider = MagicMock()
+ second_provider.get_access_token.return_value = token
+ second_provider.reload_tokens.return_value = False
+ mock_provider_cls.side_effect = [first_provider, second_provider]
+ mock_default_httpx_client.side_effect = [httpx.Client(), httpx.Client()]
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ first_client = create_client(config_path=config_path)
+ config_path = _write_config(
+ tmp_path,
+ token=token,
+ refresh_token="refresh_abc",
+ certificate_authority=second_ca,
+ )
+ second_client = create_client(config_path=config_path)
+ try:
+ assert first_client is not None
+ assert second_client is not None
+ finally:
+ second_client.close()
+ first_client.close()
+
+ assert mock_provider_cls.call_count == 2
+ certificate_authorities = [call.kwargs["certificate_authority"] for call in mock_provider_cls.call_args_list]
+ assert certificate_authorities == [first_ca, second_ca]
+
class TestCreateClientOverrides:
@patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
@@ -810,7 +972,7 @@ async def test_async_constructor_passes_context_name_to_bootstrap(self, mock_bui
class TestAsyncNeMoPlatformInit:
@pytest.mark.asyncio
- @patch("nemo_platform.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
+ @patch("nemo_platform_ext.client.factory.discover_nmp_config", return_value=_MOCK_NMP_CONFIG)
async def test_async_client_uses_config_for_api_key(self, _mock_discover, tmp_path):
config_path = _write_config(tmp_path, user_type="api-key", api_key="nvapi-test-key-123")
diff --git a/packages/nemo_platform_ext/tests/client/test_tls.py b/packages/nemo_platform_ext/tests/client/test_tls.py
new file mode 100644
index 0000000000..c7742d5d07
--- /dev/null
+++ b/packages/nemo_platform_ext/tests/client/test_tls.py
@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from nemo_platform_ext.client.tls import NMP_CLIENT_SSL_CERT_FILE_ENVVAR, httpx_tls_config_from_env
+
+
+def test_httpx_tls_config_from_env_defaults_to_certificate_validation(monkeypatch):
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ assert httpx_tls_config_from_env() == {}
+
+
+def test_httpx_tls_config_from_env_ignores_blank_ca_bundle(monkeypatch):
+ monkeypatch.setenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, " ")
+
+ assert httpx_tls_config_from_env() == {}
+
+
+def test_httpx_tls_config_from_env_uses_custom_ca_bundle(monkeypatch):
+ monkeypatch.setenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "/tmp/nemo-ca.pem")
+
+ assert httpx_tls_config_from_env() == {"verify": "/tmp/nemo-ca.pem"}
+
+
+def test_httpx_tls_config_from_env_uses_context_certificate_authority(monkeypatch):
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ assert httpx_tls_config_from_env("/tmp/context-ca.pem") == {"verify": "/tmp/context-ca.pem"}
+
+
+def test_httpx_tls_config_from_env_env_ca_bundle_overrides_context_certificate_authority(monkeypatch):
+ monkeypatch.setenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "/tmp/env-ca.pem")
+
+ assert httpx_tls_config_from_env("/tmp/context-ca.pem") == {"verify": "/tmp/env-ca.pem"}
+
+
+def test_httpx_tls_config_from_env_uses_env_overlay(monkeypatch):
+ monkeypatch.setenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "/tmp/default-ca.pem")
+
+ assert httpx_tls_config_from_env(env={NMP_CLIENT_SSL_CERT_FILE_ENVVAR: "/tmp/override-ca.pem"}) == {
+ "verify": "/tmp/override-ca.pem"
+ }
+
+
+def test_httpx_tls_config_from_env_uses_first_nonblank_configured_envvar(monkeypatch):
+ monkeypatch.delenv(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, raising=False)
+
+ assert httpx_tls_config_from_env(
+ env={
+ NMP_CLIENT_SSL_CERT_FILE_ENVVAR: " ",
+ "REQUESTS_CA_BUNDLE": "/tmp/requests-ca.pem",
+ "SSL_CERT_FILE": "/tmp/ssl-ca.pem",
+ },
+ cert_file_envvars=(NMP_CLIENT_SSL_CERT_FILE_ENVVAR, "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE"),
+ ) == {"verify": "/tmp/requests-ca.pem"}
diff --git a/packages/nemo_platform_ext/tests/config/test_config.py b/packages/nemo_platform_ext/tests/config/test_config.py
index a457b35601..4f61b656e9 100644
--- a/packages/nemo_platform_ext/tests/config/test_config.py
+++ b/packages/nemo_platform_ext/tests/config/test_config.py
@@ -35,6 +35,7 @@ def temp_config_file(tmp_path: Path) -> Path:
{
"name": "local-cluster",
"base_url": "http://localhost:8000",
+ "certificate_authority": "/tmp/local-ca.pem",
},
],
"users": [
@@ -131,6 +132,7 @@ def test_load_local_context_from_file(self, temp_config_file: Path):
assert config.context_name == "local"
assert config.cluster.name == "local-cluster"
assert str(config.cluster.base_url) == "http://localhost:8000/"
+ assert config.cluster.certificate_authority == "/tmp/local-ca.pem"
assert config.workspace == "local-workspace"
assert config.preferences.output_format == "table"
assert config.preferences.timestamp_format == "relative"
@@ -227,6 +229,45 @@ def test_sdk_override_workspace(self, temp_config_file: Path):
assert config.context_name == "production"
assert config.workspace == "sdk-workspace"
+ def test_sdk_override_certificate_authority(self, temp_config_file: Path, tmp_path: Path):
+ """Test SDK params overriding cluster certificate authority."""
+ ca_bundle = str(tmp_path / "override-ca.pem")
+ params: ConfigParams = {"current_context": "local", "certificate_authority": ca_bundle}
+
+ config = get_context(config_path=temp_config_file, overrides=params)
+
+ assert config.context_name == "local"
+ assert config.cluster.certificate_authority == ca_bundle
+
+ def test_sdk_certificate_authority_override_does_not_mutate_stored_cluster(
+ self, temp_config_file: Path, tmp_path: Path
+ ):
+ """Test runtime certificate authority override does not mutate loaded config."""
+ ca_bundle = str(tmp_path / "override-ca.pem")
+ params: ConfigParams = {"current_context": "local", "certificate_authority": ca_bundle}
+ config = Config.load(config_path=temp_config_file, overrides=params)
+
+ resolved = config.resolve()
+ stored_cluster = next(cluster for cluster in config._config_file.clusters if cluster.name == "local-cluster")
+
+ assert resolved.cluster.certificate_authority == ca_bundle
+ assert stored_cluster.certificate_authority == "/tmp/local-ca.pem"
+
+ config.certificate_authority = None
+ assert config.resolve().cluster.certificate_authority == "/tmp/local-ca.pem"
+
+ def test_reload_preserves_certificate_authority_override(self, temp_config_file: Path, tmp_path: Path):
+ """Test reload preserves runtime certificate authority override."""
+ ca_bundle = str(tmp_path / "override-ca.pem")
+ params: ConfigParams = {"current_context": "local", "certificate_authority": ca_bundle}
+ config = Config.load(config_path=temp_config_file, overrides=params)
+
+ config.reload()
+
+ resolved = config.resolve()
+ assert resolved.context_name == "local"
+ assert resolved.cluster.certificate_authority == ca_bundle
+
def test_sdk_override_preferences(self, temp_config_file: Path):
"""Test SDK params overriding preferences."""
params: ConfigParams = {
@@ -435,6 +476,25 @@ def test_config_from_sdk_params_only(self, tmp_path: Path):
assert config.workspace == "sdk-workspace"
assert config.preferences.output_format == "table"
+ def test_config_from_sdk_params_only_with_certificate_authority(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ):
+ """Test SDK certificate authority override without a config file."""
+ default_config_path = tmp_path / "missing-config.yaml"
+ monkeypatch.delenv("NMP_CONFIG_FILE", raising=False)
+ monkeypatch.setattr(Config, "get_default_config_path", classmethod(lambda _cls: default_config_path))
+ ca_bundle = str(tmp_path / "sdk-ca.pem")
+ params: ConfigParams = {
+ "base_url": "https://api.sdk.com",
+ "certificate_authority": ca_bundle,
+ }
+
+ config = Config.load(overrides=params)
+ resolved = config.resolve()
+
+ assert str(resolved.cluster.base_url) == "https://api.sdk.com/"
+ assert resolved.cluster.certificate_authority == ca_bundle
+
def test_config_without_file_no_auth(self, tmp_path: Path):
"""Test configuration without auth (NoAuth) and without config file."""
params: ConfigParams = {
diff --git a/packages/nmp_common/tests/sdk_factory/test_sdk.py b/packages/nmp_common/tests/sdk_factory/test_sdk.py
index e981cb5f53..217a4685d9 100644
--- a/packages/nmp_common/tests/sdk_factory/test_sdk.py
+++ b/packages/nmp_common/tests/sdk_factory/test_sdk.py
@@ -190,7 +190,8 @@ def token_exchange_grant(**kwargs):
monkeypatch.setenv("NMP_PRINCIPAL", json.dumps({"id": "creator@example.com", "email": "creator@example.com"}))
monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False)
monkeypatch.setattr(
- "nemo_platform_ext.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config()
+ "nemo_platform_ext.client.factory.discover_nmp_config",
+ lambda _base_url, **_kwargs: _workload_oidc_config(),
)
monkeypatch.setattr("nemo_platform_ext.auth.workload_exchange.token_exchange_grant", token_exchange_grant)
@@ -391,7 +392,8 @@ def token_exchange_grant(**kwargs):
monkeypatch.setenv("NMP_PRINCIPAL", json.dumps({"id": "creator@example.com", "email": "creator@example.com"}))
monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False)
monkeypatch.setattr(
- "nemo_platform_ext.client.factory.discover_nmp_config", lambda _base_url: _workload_oidc_config()
+ "nemo_platform_ext.client.factory.discover_nmp_config",
+ lambda _base_url, **_kwargs: _workload_oidc_config(),
)
monkeypatch.setattr("nemo_platform_ext.auth.workload_exchange.token_exchange_grant", token_exchange_grant)
diff --git a/sdk/python/nemo-platform/src/nemo_platform/_client.py b/sdk/python/nemo-platform/src/nemo_platform/_client.py
index 347a4bc80d..7dcf0bee37 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/_client.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/_client.py
@@ -226,7 +226,7 @@ def __init__(
"""
env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL")
bootstrap_base_url = base_url if base_url is not None else env_base_url
-
+ client_verify = client_verify_from_env()
should_bootstrap = _should_bootstrap_config(
http_client=http_client,
base_url=base_url,
@@ -254,6 +254,7 @@ def __init__(
):
raise TypeError("Expected httpx.Client from sync client factory")
http_client = client_init_kwargs.http_client
+ client_verify = client_init_kwargs.client_verify
except Exception as e:
raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e
@@ -263,7 +264,6 @@ def __init__(
if base_url is None:
raise RuntimeError("NeMoPlatform client initialization failed: base_url is required")
- client_verify = client_verify_from_env()
if http_client is None and client_verify is not True:
http_client = DefaultHttpxClient(verify=client_verify)
@@ -625,7 +625,7 @@ async def main() -> None:
"""
env_base_url = os.environ.get("NEMO_PLATFORM_BASE_URL")
bootstrap_base_url = base_url if base_url is not None else env_base_url
-
+ client_verify = client_verify_from_env()
should_bootstrap = _should_bootstrap_config(
http_client=http_client,
base_url=base_url,
@@ -653,6 +653,7 @@ async def main() -> None:
):
raise TypeError("Expected httpx.AsyncClient from async client factory")
http_client = client_init_kwargs.http_client
+ client_verify = client_init_kwargs.client_verify
except Exception as e:
raise RuntimeError(f"NeMoPlatform client initialization failed: {e}") from e
@@ -662,7 +663,6 @@ async def main() -> None:
if base_url is None:
raise RuntimeError("NeMoPlatform client initialization failed: base_url is required")
- client_verify = client_verify_from_env()
if http_client is None and client_verify is not True:
http_client = DefaultAsyncHttpxClient(verify=client_verify)
diff --git a/tests/auth_idp/compose/test_authentik_cli_login.py b/tests/auth_idp/compose/test_authentik_cli_login.py
index a323dfe884..fa096fb52f 100644
--- a/tests/auth_idp/compose/test_authentik_cli_login.py
+++ b/tests/auth_idp/compose/test_authentik_cli_login.py
@@ -4,7 +4,7 @@
import httpx
import pytest
from nemo_platform_ext.auth.helpers import discover_nmp_config
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import httpx_tls_config_from_env
from tests.auth_idp.authentik_live import AUTHENTIK_DOCKER_E2E_CONFIG
@@ -33,7 +33,7 @@ def test_authentik_discovery_exposes_gateway_reachable_device_flow(authentik_sta
"scope": oidc.default_scopes,
},
timeout=30.0,
- verify=client_verify_from_env(),
+ **httpx_tls_config_from_env(),
)
response.raise_for_status()
body = response.json()
@@ -55,7 +55,7 @@ def test_authentik_cli_provider_rejects_unseeded_human_app_password(authentik_st
"scope": "openid email offline_access groups",
},
timeout=30.0,
- verify=client_verify_from_env(),
+ **httpx_tls_config_from_env(),
)
assert token_response.status_code == 400
diff --git a/tests/auth_idp/conftest.py b/tests/auth_idp/conftest.py
index 03751e53a7..c3e905d639 100644
--- a/tests/auth_idp/conftest.py
+++ b/tests/auth_idp/conftest.py
@@ -25,6 +25,16 @@
pytest_plugins = ("e2e.conftest",)
+def _auth_idp_case_for_item(item: pytest.Item) -> AuthIdpCase | None:
+ callspec = getattr(item, "callspec", None)
+ if callspec is None:
+ return None
+ case = callspec.params.get("auth_idp_case")
+ if isinstance(case, AuthIdpCase):
+ return case
+ return None
+
+
def pytest_addoption(parser: pytest.Parser) -> None:
group = parser.getgroup("auth-idp")
group.addoption(
@@ -72,6 +82,11 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item
runtime_markers = list(item.iter_markers("auth_idp_runtime"))
if runtime_markers:
item.add_marker(pytest.mark.xdist_group("idp-live"))
+ case = _auth_idp_case_for_item(item)
+ if case is not None and case.backend == "kubernetes" and item.get_closest_marker("timeout") is None:
+ from tests.auth_idp.runtime_kubernetes import PYTEST_TIMEOUT_SECONDS
+
+ item.add_marker(pytest.mark.timeout(PYTEST_TIMEOUT_SECONDS))
if not runtime_markers:
selected.append(item)
continue
diff --git a/tests/auth_idp/k8s/test_authentik_kubernetes_live.py b/tests/auth_idp/k8s/test_authentik_kubernetes_live.py
index 3cee2bc5ea..d85679a7bb 100644
--- a/tests/auth_idp/k8s/test_authentik_kubernetes_live.py
+++ b/tests/auth_idp/k8s/test_authentik_kubernetes_live.py
@@ -5,9 +5,12 @@
from tests.auth_idp.runtime_factory import iter_auth_idp_cases
from tests.auth_idp.runtime_kubernetes import (
+ HELM_UPGRADE_COMMAND_GRACE_SECONDS,
HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS,
+ HELM_WAIT_TIMEOUT,
PORT_FORWARD_READY_TIMEOUT_SECONDS,
_add_platform_helm_repositories,
+ _duration_seconds,
_helm_upgrade_args,
)
@@ -24,5 +27,8 @@
def test_authentik_kubernetes_runtime_exports_helm_contract_helpers() -> None:
assert _helm_upgrade_args
assert _add_platform_helm_repositories
- assert HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS == 900
+ assert (
+ HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS
+ == _duration_seconds(HELM_WAIT_TIMEOUT) + HELM_UPGRADE_COMMAND_GRACE_SECONDS
+ )
assert PORT_FORWARD_READY_TIMEOUT_SECONDS == 30
diff --git a/tests/auth_idp/runtime_compose.py b/tests/auth_idp/runtime_compose.py
index a3f27fb524..4a0fcbf69a 100644
--- a/tests/auth_idp/runtime_compose.py
+++ b/tests/auth_idp/runtime_compose.py
@@ -7,7 +7,7 @@
import httpx
from nemo_platform import NeMoPlatform
-from nemo_platform_ext.client.tls import client_verify_from_env
+from nemo_platform_ext.client.tls import httpx_tls_config_from_env
from tests.auth_idp.common import jwt_claims
from tests.auth_idp.runtime_contract import AuthIdpCase, TokenSet
@@ -89,7 +89,7 @@ def exchange_workload_token(self, subject_token: str) -> TokenSet:
"scope": workload_grant.get("scope", "openid email groups"),
},
timeout=TOKEN_EXCHANGE_TIMEOUT_SECONDS,
- verify=client_verify_from_env(),
+ **httpx_tls_config_from_env(),
)
response.raise_for_status()
token_response = response.json()
diff --git a/tests/auth_idp/runtime_kubernetes.py b/tests/auth_idp/runtime_kubernetes.py
index ccfdce3d77..965092275e 100644
--- a/tests/auth_idp/runtime_kubernetes.py
+++ b/tests/auth_idp/runtime_kubernetes.py
@@ -7,6 +7,7 @@
import contextlib
import json
import os
+import re
import shutil
import socket
import subprocess
@@ -44,8 +45,29 @@
T = TypeVar("T")
+_DURATION_TOKEN_RE = re.compile(r"(\d+)([hms])")
+
+
+def _duration_seconds(value: str) -> int:
+ if value.isdigit():
+ return int(value)
+
+ total = 0
+ position = 0
+ for match in _DURATION_TOKEN_RE.finditer(value):
+ if match.start() != position:
+ raise ValueError(f"invalid duration: {value}")
+ amount, unit = match.groups()
+ total += int(amount) * {"h": 3600, "m": 60, "s": 1}[unit]
+ position = match.end()
+
+ if position != len(value) or total == 0:
+ raise ValueError(f"invalid duration: {value}")
+ return total
+
+
# Keep timeouts centralized so slow-cluster tuning is a single, visible edit.
-PYTEST_TIMEOUT_SECONDS = 900
+PYTEST_TIMEOUT_SECONDS = 2400
DEFAULT_COMMAND_TIMEOUT_SECONDS = 120
DIAGNOSTIC_COMMAND_TIMEOUT_SECONDS = 60
POD_DISCOVERY_TIMEOUT_SECONDS = 60
@@ -55,8 +77,9 @@
CLUSTER_DELETE_TIMEOUT_SECONDS = 180
ROLLOUT_STATUS_TIMEOUT = "240s"
ROLLOUT_COMMAND_TIMEOUT_SECONDS = 300
-HELM_WAIT_TIMEOUT = "10m"
-HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS = 900
+HELM_WAIT_TIMEOUT = os.environ.get("NMP_AUTHENTIK_K8S_HELM_WAIT_TIMEOUT", "20m")
+HELM_UPGRADE_COMMAND_GRACE_SECONDS = 300
+HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS = _duration_seconds(HELM_WAIT_TIMEOUT) + HELM_UPGRADE_COMMAND_GRACE_SECONDS
HELM_REPO_TIMEOUT_SECONDS = 60
HELM_DEPENDENCY_TIMEOUT_SECONDS = 300
HTTP_RETRY_TIMEOUT_SECONDS = 180
@@ -169,6 +192,13 @@ def _diagnostic_output_text(output: str | bytes | None) -> str:
return output
+def _diagnostic_text_tail(path: Path, *, limit: int = 4000) -> str:
+ try:
+ return path.read_text(encoding="utf-8", errors="replace")[-limit:]
+ except OSError:
+ return ""
+
+
def _write_diagnostic_timeout(log_dir: Path, name: str, args: list[str], exc: subprocess.TimeoutExpired) -> None:
(log_dir / name).write_text(
"\n".join(
@@ -197,6 +227,24 @@ def _write_diagnostic_command(
_write_diagnostic_process(log_dir, name, _kubectl_command(context, args, kubeconfig), timeout=timeout)
+def _port_forward_log_file(service: str) -> Path | None:
+ configured_dir = os.environ.get("NMP_AUTHENTIK_K8S_LOG_DIR")
+ if not configured_dir:
+ return None
+ safe_service = service.replace("/", "_")
+ return Path(configured_dir) / f"port-forward-{safe_service}.log"
+
+
+def _port_forward_exit_message(returncode: int, log_file: Path | None) -> str:
+ message = f"kubectl port-forward exited early with {returncode}"
+ if log_file is None:
+ return message
+ log_tail = _diagnostic_text_tail(log_file)
+ if not log_tail.strip():
+ return message
+ return f"{message}\nport-forward log ({log_file}):\n{log_tail}"
+
+
def _collect_kubernetes_diagnostics(context: str, cluster_name: str, kubeconfig: Path | None = None) -> Path:
configured_dir = os.environ.get("NMP_AUTHENTIK_K8S_LOG_DIR")
log_root = Path(configured_dir) if configured_dir else REPO_ROOT / "docker" / "logs"
@@ -662,28 +710,43 @@ def _start_port_forward_service(
kubeconfig: Path | None = None,
) -> tuple[str, subprocess.Popen[str]]:
port = _configured_gateway_port() or _free_port()
- process = subprocess.Popen(
- _kubectl_command(
- context,
- [
- "-n",
- NAMESPACE,
- "port-forward",
- f"svc/{service}",
- f"{port}:8080",
- ],
- kubeconfig,
- ),
- cwd=REPO_ROOT,
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- text=True,
+ command = _kubectl_command(
+ context,
+ [
+ "-n",
+ NAMESPACE,
+ "port-forward",
+ f"svc/{service}",
+ f"{port}:8080",
+ ],
+ kubeconfig,
)
+ log_file = _port_forward_log_file(service)
+ log_handle = None
+ stdout = subprocess.DEVNULL
+ if log_file is not None:
+ log_file.parent.mkdir(parents=True, exist_ok=True)
+ log_handle = log_file.open("w", encoding="utf-8")
+ log_handle.write(f"command: {' '.join(command)}\n\n")
+ log_handle.flush()
+ stdout = log_handle
+ try:
+ process = subprocess.Popen(
+ command,
+ cwd=REPO_ROOT,
+ stdout=stdout,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ finally:
+ if log_handle is not None:
+ log_handle.close()
gateway_url = f"https://127.0.0.1:{port}"
def wait_for_gateway_ready(remaining: float) -> httpx.Response:
if process.poll() is not None:
- raise AssertionError(f"kubectl port-forward exited early with {process.returncode}")
+ assert process.returncode is not None
+ raise AssertionError(_port_forward_exit_message(process.returncode, log_file))
return httpx.get(
gateway_url + GATEWAY_READY_PATH,
timeout=min(PORT_FORWARD_HTTP_TIMEOUT_SECONDS, remaining),
diff --git a/tests/auth_idp/static/test_authentik_kubernetes_demo.py b/tests/auth_idp/static/test_authentik_kubernetes_demo.py
index 4dfcdb80ff..3ca0b60f04 100644
--- a/tests/auth_idp/static/test_authentik_kubernetes_demo.py
+++ b/tests/auth_idp/static/test_authentik_kubernetes_demo.py
@@ -4,6 +4,7 @@
import ast
import importlib.util
import os
+import re
import shutil
import subprocess
import sys
@@ -119,25 +120,13 @@ def _run_authentik_script(*args: str, env: dict[str, str] | None = None) -> str:
return completed.stdout
-def test_authentik_run_local_defaults_workload_identity_password_for_compose() -> None:
- env = os.environ.copy()
- env.pop("AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD", None)
- completed = subprocess.run(
- [str(AUTHENTIK_DIR / "run.sh"), "run-local", "--dry-run"],
- text=True,
- capture_output=True,
- check=False,
- env=env,
- timeout=AUTHENTIK_SCRIPT_TIMEOUT_SECONDS,
+def _gateway_port_from_script_output(output: str) -> str:
+ match = re.search(
+ r"\b(?:NMP_AUTHENTIK_K8S_GATEWAY_PORT|nemo-platform\.authentikPublicGateway\.port)=(\d+)\b",
+ output,
)
-
- assert completed.returncode == 0, completed.stdout + completed.stderr
- assert "helm/files/blueprints/nemo.yaml" in completed.stdout
- assert "contrib/auth/authentik/.generated/workload-token-private-key.pem" in completed.stdout
- assert "contrib/auth/authentik/.generated/gateway-tls" in completed.stdout
- assert "contrib/auth/authentik/compose" in completed.stdout
- assert "AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD=" in completed.stdout
- assert "docker compose up" in completed.stdout
+ assert match is not None, output
+ return match.group(1)
def test_authentik_prepare_local_creates_shared_generated_inputs() -> None:
@@ -178,7 +167,6 @@ def test_authentik_user_startup_docs_use_manual_runtime_steps() -> None:
"--set-file workloadTokenSigningKey.privateKeyPem="
"contrib/auth/authentik/.generated/workload-token-private-key.pem"
) in tutorial
- assert "contrib/auth/authentik/run.sh run-local" not in tutorial
assert "contrib/auth/authentik/run.sh compose" not in tutorial
assert "contrib/auth/authentik/run.sh k8s" not in tutorial
assert "run.sh" not in compose_readme
@@ -341,6 +329,14 @@ def test_authentik_umbrella_values_use_latest_authentik_chart_without_image_tag_
assert authentik_values.get("global", {}).get("image", {}).get("tag", "") == ""
+def test_authentik_umbrella_values_extend_local_startup_probe_budget() -> None:
+ values = _load_yaml(HELM_DIR / "values.yaml")
+ nemo_values = values["nemo-platform"]
+
+ assert nemo_values["api"]["startupProbe"] == {"failureThreshold": 80}
+ assert nemo_values["core"]["controller"]["startupProbe"] == {"failureThreshold": 80}
+
+
def test_authentik_umbrella_values_define_one_shared_postgresql_instance() -> None:
values = _load_yaml(HELM_DIR / "values.yaml")
initdb_template = (HELM_DIR / "templates" / "shared-postgres-initdb-configmap.yaml").read_text(encoding="utf-8")
@@ -421,11 +417,30 @@ def test_authentik_kubernetes_live_timeouts_are_named_constants() -> None:
args = live_test._helm_upgrade_args("kind-ci")
assert args[args.index("--timeout") + 1] == live_test.HELM_WAIT_TIMEOUT
- assert live_test.HELM_WAIT_TIMEOUT == "10m"
- assert live_test.HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS == 900
+ assert live_test.PYTEST_TIMEOUT_SECONDS == 2400
+ assert live_test.HELM_WAIT_TIMEOUT == "20m"
+ assert live_test.HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS == (
+ live_test._duration_seconds(live_test.HELM_WAIT_TIMEOUT) + live_test.HELM_UPGRADE_COMMAND_GRACE_SECONDS
+ )
assert live_test.PORT_FORWARD_READY_TIMEOUT_SECONDS == 30
+def test_authentik_kubernetes_helm_command_timeout_tracks_wait_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("NMP_AUTHENTIK_K8S_HELM_WAIT_TIMEOUT", "1h5m30s")
+
+ live_test = _load_authentik_k8s_live_module()
+
+ assert live_test.HELM_WAIT_TIMEOUT == "1h5m30s"
+ assert live_test.HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS == 4230
+
+
+def test_authentik_kubernetes_contract_tests_get_runtime_timeout() -> None:
+ auth_idp_conftest = Path("tests/auth_idp/conftest.py").read_text(encoding="utf-8")
+
+ assert 'case.backend == "kubernetes"' in auth_idp_conftest
+ assert "pytest.mark.timeout(PYTEST_TIMEOUT_SECONDS)" in auth_idp_conftest
+
+
def test_authentik_kubernetes_reuse_context_validates_runtime() -> None:
live_test = _load_authentik_k8s_live_module()
@@ -603,6 +618,41 @@ def fake_get(url: str, **_kwargs: object):
assert requested_urls == ["https://127.0.0.1:19001/health/gateway/ready"]
+def test_authentik_kubernetes_port_forward_reports_process_log(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ live_test = _load_authentik_k8s_live_module()
+ monkeypatch.setenv("NMP_AUTHENTIK_K8S_LOG_DIR", str(tmp_path))
+ monkeypatch.setattr(live_test, "_free_port", lambda: 19001)
+
+ class FakeProcess:
+ returncode = 1
+
+ def poll(self) -> int:
+ return self.returncode
+
+ def terminate(self) -> None:
+ return None
+
+ def wait(self, timeout: int | None = None) -> int:
+ return self.returncode
+
+ def kill(self) -> None:
+ return None
+
+ def fake_popen(*_args: object, **kwargs: object) -> FakeProcess:
+ stdout = kwargs["stdout"]
+ getattr(stdout, "write")("error: unable to listen on any of the requested ports: address already in use\n")
+ getattr(stdout, "flush")()
+ return FakeProcess()
+
+ monkeypatch.setattr(live_test.subprocess, "Popen", fake_popen)
+
+ with pytest.raises(AssertionError, match="address already in use"):
+ live_test._start_port_forward_service("kind-ci", "nemo-platform-envoy", Path("ca.crt"))
+
+ log_file = tmp_path / "port-forward-nemo-platform-envoy.log"
+ assert "kubectl --context kind-ci -n nemo-authentik port-forward" in log_file.read_text(encoding="utf-8")
+
+
def test_authentik_kubernetes_port_forward_waits_after_kill(monkeypatch: pytest.MonkeyPatch) -> None:
live_test = _load_authentik_k8s_live_module()
events: list[object] = []
@@ -899,6 +949,8 @@ def test_authentik_umbrella_values_configure_nemo_envoy_as_the_only_edge_proxy()
assert nemo_values["envoyProxy"]["serviceNamespace"] == ""
assert values["integration"]["nemoPlatform"]["envoyServiceName"] == "nemo-platform-envoy"
assert nemo_values["rbac"]["volcanoEnabled"] is False
+ assert nemo_values["api"]["startupProbe"] == {"failureThreshold": 80}
+ assert nemo_values["core"]["controller"]["startupProbe"] == {"failureThreshold": 80}
assert nemo_values["platformConfig"]["models"]["controller"]["backends"] == {
"deployments_plugin": {"enabled": True},
}
@@ -1029,10 +1081,13 @@ def test_authentik_kubernetes_runner_uses_helm_not_kustomize() -> None:
assert "nemo-platform.imagePullSecrets[0].name=" in runtime_impl
assert "NMP_AUTHENTIK_K8S_NGC_EXISTING_SECRET" in runtime_impl
assert "nemo-platform.existingSecret=" in runtime_impl
- assert 'K8S_GATEWAY_PORT="${NMP_AUTHENTIK_K8S_GATEWAY_PORT:-18082}"' in run_sh
+ assert 'DEFAULT_K8S_GATEWAY_PORT="18082"' in run_sh
+ assert 'K8S_GATEWAY_PORT="${NMP_AUTHENTIK_K8S_GATEWAY_PORT:-}"' in run_sh
+ assert "choose_free_tcp_port" in run_sh
assert "NMP_AUTHENTIK_K8S_GATEWAY_PORT=${K8S_GATEWAY_PORT}" in run_sh
assert "NMP_AUTHENTIK_K8S_GATEWAY_PORT" in runtime_impl
assert "nemo-platform.authentikPublicGateway.port=" in runtime_impl
+ assert "_port_forward_log_file" in runtime_impl
assert "nemo-platform.platformConfig.auth.access_keys.enabled=true" in runtime_impl
assert "GITHUB_TOKEN: ${{ inputs['kind-image-pull-token'] }}" in setup_kind_action
assert "CERT_MANAGER_CHART" not in runtime_impl
@@ -1041,7 +1096,11 @@ def test_authentik_kubernetes_runner_uses_helm_not_kustomize() -> None:
assert ("helm", "repo", "add", "authentik", "https://charts.goauthentik.io", "--force-update") in run_commands
assert 'os.environ.get("NMP_AUTHENTIK_K8S_RUNTIME", "kind")' in runtime_impl
assert '"--no-hooks"' not in runtime_impl
- assert "HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS = 900" in runtime_impl
+ assert "HELM_UPGRADE_COMMAND_GRACE_SECONDS = 300" in runtime_impl
+ assert (
+ "HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS = _duration_seconds(HELM_WAIT_TIMEOUT) "
+ "+ HELM_UPGRADE_COMMAND_GRACE_SECONDS"
+ ) in runtime_impl
assert "_run(_helm_upgrade_args(context, kubeconfig), timeout=HELM_UPGRADE_COMMAND_TIMEOUT_SECONDS)" in runtime_impl
assert "PORT_FORWARD_READY_TIMEOUT_SECONDS = 30" in runtime_impl
assert "certificates.cert-manager.io" not in runtime_impl
@@ -1055,6 +1114,7 @@ def test_authentik_kubernetes_runner_uses_helm_not_kustomize() -> None:
def test_authentik_kubernetes_runner_builds_when_only_image_tag_env_is_set() -> None:
output = _run_authentik_script(
+ "test",
"k8s",
"--dry-run",
env={
@@ -1071,7 +1131,7 @@ def test_authentik_kubernetes_runner_builds_when_only_image_tag_env_is_set() ->
def test_authentik_compose_runner_reuse_uses_stable_project_and_port() -> None:
- output = _run_authentik_script("compose", "--dry-run", "--reuse")
+ output = _run_authentik_script("test", "compose", "--dry-run", "--reuse")
assert "workload-token-private-key.pem" in output
assert "NMP_E2E_COMPOSE_LIFECYCLE=reuse" in output
@@ -1080,35 +1140,268 @@ def test_authentik_compose_runner_reuse_uses_stable_project_and_port() -> None:
assert "--auth-idp-runtime authentik-compose" in output
+def test_authentik_compose_up_starts_reusable_stack_without_pytest() -> None:
+ output = _run_authentik_script("up", "compose", "--dry-run")
+
+ assert "COMPOSE_PROJECT_NAME=authentik-e2e-reuse" in output
+ assert "AUTHENTIK_GATEWAY_PORT=18083" in output
+ assert "docker compose up -d" in output
+ assert "https://127.0.0.1:18083/health/gateway/ready" in output
+ assert "uv run --frozen nemo config set --context authentik-compose" in output
+ assert "--certificate-authority" in output
+ assert "write lifecycle state" in output
+ assert "uv run --frozen pytest" not in output
+ assert "--auth-idp-runtime authentik-compose" not in output
+
+
+def test_authentik_compose_up_key_derives_managed_instance_names() -> None:
+ output = _run_authentik_script(
+ "up",
+ "compose",
+ "--key",
+ "dev",
+ "--dry-run",
+ env={"NMP_AUTHENTIK_COMPOSE_GATEWAY_PORT": "19083"},
+ )
+
+ assert "COMPOSE_PROJECT_NAME=authentik-e2e-dev" in output
+ assert "AUTHENTIK_GATEWAY_PORT=19083" in output
+ assert "AUTHENTIK_GATEWAY_TLS_VOLUME=authentik-e2e-dev-gateway-tls" in output
+ assert "AUTHENTIK_WORKLOAD_NETWORK_NAME=authentik-e2e-dev-workload" in output
+ assert "uv run --frozen nemo config set --context authentik-compose-dev" in output
+ assert "write lifecycle state" in output
+ assert "run.sh down compose --key dev" in output
+
+
+def test_authentik_compose_test_action_runs_contract_pytest() -> None:
+ output = _run_authentik_script("test", "compose", "--dry-run")
+
+ assert "uv run --frozen pytest tests/auth_idp/contracts" in output
+ assert "--auth-idp-runtime authentik-compose" in output
+
+
+def test_authentik_runner_rejects_removed_bare_compose_and_k8s_aliases() -> None:
+ for target in ("compose", "k8s"):
+ completed = subprocess.run(
+ [str(AUTHENTIK_DIR / "run.sh"), target, "--dry-run"],
+ text=True,
+ capture_output=True,
+ check=False,
+ timeout=AUTHENTIK_SCRIPT_TIMEOUT_SECONDS,
+ )
+
+ assert completed.returncode == 2
+ assert f"unknown argument: {target}" in completed.stderr
+
+
+def test_authentik_ci_workflow_uses_explicit_test_actions() -> None:
+ workflow = _load_yaml(Path(".github/workflows/ci.yaml"))
+ matrix_include = workflow["jobs"]["python-auth-idp-e2e-test"]["strategy"]["matrix"]["include"]
+
+ commands_by_runtime = {
+ entry["runtime"]: entry["command"] for entry in matrix_include if entry.get("provider") == "authentik"
+ }
+
+ assert commands_by_runtime == {
+ "authentik-compose": "test compose",
+ "authentik-kubernetes": "test k8s",
+ }
+
+
def test_authentik_kubernetes_runner_reuse_uses_stable_cluster() -> None:
- output = _run_authentik_script("k8s", "--dry-run", "--reuse")
+ output = _run_authentik_script("test", "k8s", "--dry-run", "--reuse")
assert "NMP_AUTHENTIK_K8S_CLUSTER_NAME=nmp-authentik-reuse" in output
- assert "NMP_AUTHENTIK_K8S_GATEWAY_PORT=18082" in output
+ assert _gateway_port_from_script_output(output)
assert "NMP_AUTHENTIK_K8S_REUSE_CLUSTER=1" in output
assert "NMP_AUTHENTIK_K8S_KEEP_CLUSTER=1" in output
assert "--auth-idp-runtime authentik-kubernetes" in output
-def test_authentik_down_cleans_reused_compose_and_kubernetes_resources() -> None:
- output = _run_authentik_script("down", "--dry-run")
+def test_authentik_kubernetes_test_action_chooses_dynamic_gateway_port_by_default(tmp_path: Path) -> None:
+ fake_bin = tmp_path / "bin"
+ fake_bin.mkdir()
+ fake_python = fake_bin / "python3"
+ fake_python.write_text("#!/usr/bin/env bash\nprintf '19082\\n'\n", encoding="utf-8")
+ fake_python.chmod(0o755)
+ env = os.environ.copy()
+ env.pop("NMP_AUTHENTIK_K8S_GATEWAY_PORT", None)
+ env["PATH"] = f"{fake_bin}{os.pathsep}{env['PATH']}"
+ completed = subprocess.run(
+ [str(AUTHENTIK_DIR / "run.sh"), "test", "k8s", "--dry-run"],
+ text=True,
+ capture_output=True,
+ check=False,
+ env=env,
+ timeout=AUTHENTIK_SCRIPT_TIMEOUT_SECONDS,
+ )
+
+ assert completed.returncode == 0, completed.stdout + completed.stderr
+ assert _gateway_port_from_script_output(completed.stdout) == "19082"
+
+
+def test_authentik_kubernetes_test_action_honors_explicit_gateway_port() -> None:
+ output = _run_authentik_script(
+ "test",
+ "k8s",
+ "--dry-run",
+ env={"NMP_AUTHENTIK_K8S_GATEWAY_PORT": "19082"},
+ )
+
+ assert "NMP_AUTHENTIK_K8S_GATEWAY_PORT=19082" in output
+
+
+def test_authentik_kubernetes_up_starts_reusable_stack_without_pytest() -> None:
+ output = _run_authentik_script("up", "k8s", "--dry-run", "--skip-image-load")
+ gateway_port = _gateway_port_from_script_output(output)
+
+ assert "kind create cluster --name nmp-authentik-reuse" in output
+ assert "kind export kubeconfig --name nmp-authentik-reuse" in output
+ assert "kubectl config use-context kind-nmp-authentik-reuse" in output
+ assert "helm --kubeconfig" in output
+ assert "upgrade --install authentik-demo" in output
+ assert "--timeout 20m" in output
+ assert f"nemo-platform.authentikPublicGateway.port={gateway_port}" in output
+ assert f"port-forward svc/nemo-platform-envoy {gateway_port}:8080" in output
+ assert f"https://127.0.0.1:{gateway_port}/health/gateway/ready" in output
+ assert "uv run --frozen nemo config set --context authentik-k8s" in output
+ assert "--certificate-authority" in output
+ assert "write lifecycle state" in output
+ assert "uv run --frozen pytest" not in output
+ assert "--auth-idp-runtime authentik-kubernetes" not in output
+
+
+def test_authentik_kubernetes_up_key_derives_managed_instance_names() -> None:
+ output = _run_authentik_script(
+ "up",
+ "k8s",
+ "--key",
+ "dev",
+ "--dry-run",
+ "--skip-image-load",
+ env={"NMP_AUTHENTIK_K8S_GATEWAY_PORT": "19084"},
+ )
+
+ assert "kind create cluster --name nmp-authentik-dev" in output
+ assert "kubectl config use-context kind-nmp-authentik-dev" in output
+ assert "nemo-platform.authentikPublicGateway.port=19084" in output
+ assert "port-forward svc/nemo-platform-envoy 19084:8080" in output
+ assert "uv run --frozen nemo config set --context authentik-k8s-dev" in output
+ assert "write lifecycle state" in output
+ assert "run.sh down k8s --key dev" in output
+
+
+def test_authentik_kubernetes_up_updates_default_k3d_kubeconfig() -> None:
+ output = _run_authentik_script("up", "k8s", "--dry-run", "--runtime", "k3d", "--skip-image-load")
+
+ assert "k3d kubeconfig merge nmp-authentik-reuse --kubeconfig-merge-default --kubeconfig-switch-context" in output
+ assert "kubectl config use-context k3d-nmp-authentik-reuse" in output
+
+
+def test_authentik_kubernetes_test_action_runs_contract_pytest() -> None:
+ output = _run_authentik_script("test", "k8s", "--dry-run")
+
+ assert "NMP_AUTHENTIK_K8S_HELM_WAIT_TIMEOUT=20m" in output
+ assert "uv run --frozen pytest tests/auth_idp/contracts" in output
+ assert "--auth-idp-runtime authentik-kubernetes" in output
+
+
+def test_authentik_down_cleans_reused_compose_and_kubernetes_resources(tmp_path: Path) -> None:
+ output = _run_authentik_script("down", "--dry-run", env={"NEMO_AUTHENTIK_STATE_DIR": str(tmp_path)})
assert "docker compose down -v --remove-orphans" in output
assert "COMPOSE_PROJECT_NAME=authentik-e2e-reuse" in output
assert "AUTHENTIK_GATEWAY_PORT=18083" in output
assert "AUTHENTIK_GATEWAY_TLS_VOLUME=authentik-e2e-18083-gateway-tls" in output
assert "AUTHENTIK_WORKLOAD_NETWORK_NAME=authentik-e2e-18083-workload" in output
+ assert "port-forward.pid" in output
+ assert "kind delete cluster --name nmp-authentik-reuse" in output
+ assert "nemo config delete-context authentik-compose --prune-orphans" in output
+ assert "nemo config delete-context authentik-k8s --prune-orphans" in output
+
+
+def test_authentik_down_compose_only_cleans_compose_resources(tmp_path: Path) -> None:
+ output = _run_authentik_script("down", "compose", "--dry-run", env={"NEMO_AUTHENTIK_STATE_DIR": str(tmp_path)})
+
+ assert "docker compose down -v --remove-orphans" in output
+ assert "nemo config delete-context authentik-compose --prune-orphans" in output
+ assert "kind delete cluster" not in output
+ assert "port-forward.pid" not in output
+ assert "authentik-k8s" not in output
+
+
+def test_authentik_down_k8s_only_cleans_kubernetes_resources(tmp_path: Path) -> None:
+ output = _run_authentik_script("down", "k8s", "--dry-run", env={"NEMO_AUTHENTIK_STATE_DIR": str(tmp_path)})
+
assert "kind delete cluster --name nmp-authentik-reuse" in output
+ assert "port-forward.pid" in output
+ assert "nemo config delete-context authentik-k8s --prune-orphans" in output
+ assert "docker compose down" not in output
+ assert "authentik-compose" not in output
-def test_authentik_down_accepts_kubernetes_runtime_for_reuse_cleanup() -> None:
- output = _run_authentik_script("down", "--dry-run", "--runtime", "k3d")
+def test_authentik_kubernetes_port_forward_pid_reuse_checks_process_command() -> None:
+ run_sh = (AUTHENTIK_DIR / "run.sh").read_text(encoding="utf-8")
+
+ assert "k8s_port_forward_pid_is_running" in run_sh
+ assert 'local expected_port="${2:-}"' in run_sh
+ assert 'ps -p "${pid}" -o args=' in run_sh
+ assert '[[ "${args}" == *"kubectl"* ]]' in run_sh
+ assert '[[ "${args}" == *"port-forward"* ]]' in run_sh
+ assert '[[ "${args}" == *"svc/nemo-platform-envoy"* ]]' in run_sh
+ assert '[[ "${args}" == *"${expected_port}:8080"* ]]' in run_sh
+ assert 'k8s_port_forward_pid_is_running "${pid}" "${K8S_GATEWAY_PORT}"' in run_sh
+
+
+def test_authentik_kubernetes_ca_bundle_requires_secret_entry() -> None:
+ run_sh = (AUTHENTIK_DIR / "run.sh").read_text(encoding="utf-8")
+
+ assert "has no ca.crt entry" in run_sh
+
+
+def test_authentik_kubernetes_port_forward_reports_log_on_readiness_failure() -> None:
+ run_sh = (AUTHENTIK_DIR / "run.sh").read_text(encoding="utf-8")
+
+ assert "show_k8s_port_forward_log_tail" in run_sh
+ assert 'tail -n 40 "${log_file}"' in run_sh
+ assert "timed out waiting for Kubernetes gateway port-forward readiness" in run_sh
+
+
+def test_authentik_down_key_cleans_derived_compose_and_kubernetes_contexts(tmp_path: Path) -> None:
+ output = _run_authentik_script("down", "--key", "dev", "--dry-run", env={"NEMO_AUTHENTIK_STATE_DIR": str(tmp_path)})
+
+ assert "COMPOSE_PROJECT_NAME=authentik-e2e-dev" in output
+ assert "AUTHENTIK_GATEWAY_TLS_VOLUME=authentik-e2e-dev-gateway-tls" in output
+ assert "AUTHENTIK_WORKLOAD_NETWORK_NAME=authentik-e2e-dev-workload" in output
+ assert "AUTHENTIK_WORKLOAD_IDENTITY_PASSWORD= docker compose down" not in output
+ assert "kind delete cluster --name nmp-authentik-dev" in output
+ assert "nemo config delete-context authentik-compose-dev --prune-orphans" in output
+ assert "nemo config delete-context authentik-k8s-dev --prune-orphans" in output
+
+
+def test_authentik_key_rejects_names_that_are_not_kubernetes_safe() -> None:
+ completed = subprocess.run(
+ [str(AUTHENTIK_DIR / "run.sh"), "up", "compose", "--key", "Dev_1", "--dry-run"],
+ text=True,
+ capture_output=True,
+ check=False,
+ timeout=AUTHENTIK_SCRIPT_TIMEOUT_SECONDS,
+ )
+
+ assert completed.returncode == 2
+ assert "--key must use 1-32 lowercase letters" in completed.stderr
+
+
+def test_authentik_down_accepts_kubernetes_runtime_for_reuse_cleanup(tmp_path: Path) -> None:
+ output = _run_authentik_script(
+ "down", "--dry-run", "--runtime", "k3d", env={"NEMO_AUTHENTIK_STATE_DIR": str(tmp_path)}
+ )
assert "k3d cluster delete nmp-authentik-reuse" in output
def test_authentik_kubernetes_runner_skips_build_only_for_explicit_image() -> None:
- output = _run_authentik_script("k8s", "--dry-run", "--image", "registry.example.test/nmp-api:prebuilt")
+ output = _run_authentik_script("test", "k8s", "--dry-run", "--image", "registry.example.test/nmp-api:prebuilt")
assert "Using prebuilt auth-idp Kubernetes test image: registry.example.test/nmp-api:prebuilt" in output
assert "make docker-load DOCKER_TARGET=nmp-api-docker" not in output