diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 228a71946..31d51428b 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -19,6 +19,8 @@ jobs: E2E_OPENAI_MODEL: ${{ vars.E2E_OPENAI_MODEL }} E2E_DEPLOYMENT_MODE: ${{ matrix.mode }} FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }} + # Override via repo Actions variable E2E_LLAMA_HOSTNAME; default matches server-mode lightspeed-stack.yaml + E2E_LLAMA_HOSTNAME: ${{ vars.E2E_LLAMA_HOSTNAME || 'llama-stack' }} steps: - uses: actions/checkout@v4 diff --git a/docker-compose.yaml b/docker-compose.yaml index 7c58708a1..63737f696 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -70,6 +70,8 @@ services: - LLAMA_STACK_LOGGING=${LLAMA_STACK_LOGGING:-} # FAISS test - FAISS_VECTOR_STORE_ID=${FAISS_VECTOR_STORE_ID:-} + #required for enrichment + - E2E_LLAMA_HOSTNAME=${E2E_LLAMA_HOSTNAME:-llama-stack} networks: - lightspeednet healthcheck: @@ -98,6 +100,9 @@ services: - CLIENT_SECRET=${CLIENT_SECRET:-} # FAISS vector store ID (used by inline RAG config) - FAISS_VECTOR_STORE_ID=${FAISS_VECTOR_STORE_ID:-} + # Substituted in mounted lightspeed-stack.yaml (llama_stack.url); GitHub Actions sets via vars + - E2E_LLAMA_HOSTNAME=${E2E_LLAMA_HOSTNAME:-llama-stack} + - E2E_LLAMA_PORT=${E2E_LLAMA_PORT:-8321} depends_on: llama-stack: condition: service_healthy diff --git a/docs/e2e_testing.md b/docs/e2e_testing.md index 7aeb1c409..600184a99 100644 --- a/docs/e2e_testing.md +++ b/docs/e2e_testing.md @@ -75,14 +75,7 @@ tests/e2e-prow/ ├── pipeline-services-konflux.sh # Services for Konflux (llama-stack-openai + templated LCS) ├── pipeline-vllm.sh # vLLM cluster setup (called from pipeline.sh) ├── pipeline-test-pod.sh # Test pod pipeline - ├── configs/ # Lightspeed-stack configs for Prow (used by environment.py when is_prow) - │ ├── lightspeed-stack.yaml - │ ├── lightspeed-stack-auth-noop-token.yaml - │ ├── lightspeed-stack-rbac.yaml - │ ├── lightspeed-stack-auth-rh-identity.yaml - │ ├── lightspeed-stack-no-cache.yaml - │ ├── lightspeed-stack-invalid-feedback-storage.yaml - │ └── run.yaml # vLLM Llama Stack config (used by pipeline.sh); Konflux uses tests/e2e/configs/run-ci.yaml via pipeline-konflux.sh + ├── configs/ # vLLM Llama Stack `run.yaml` (used by pipeline.sh for llama-stack-config) ├── scripts/ │ ├── e2e-ops.sh # E2E ops (e.g. disrupt/restore llama-stack) — called from prow_utils │ ├── bootstrap.sh diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml deleted file mode 100644 index 4dfd3ed4e..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - # Uses a remote llama-stack service - # The instance would have already been started with a llama-stack-run.yaml file - use_as_library_client: false - # Alternative for "as library use" - # use_as_library_client: true - # library_client_config_path: - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -# Conversation cache for storing Q&A history -conversation_cache: - type: "sqlite" - sqlite: - db_path: "/tmp/data/conversation-cache.db" - -authentication: - module: "noop-with-token" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-rh-identity.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-rh-identity.yaml deleted file mode 100644 index e2b468cf0..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-rh-identity.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) - RH Identity Auth -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: true - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -conversation_cache: - type: "sqlite" - sqlite: - db_path: "/tmp/data/conversation-cache.db" -authentication: - module: "rh-identity" - rh_identity_config: - required_entitlements: ["rhel"] diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-inline-rag.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-inline-rag.yaml deleted file mode 100644 index bdbf2b73e..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-inline-rag.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -conversation_cache: - type: "sqlite" - sqlite: - db_path: "/tmp/data/conversation-cache.db" - -authentication: - module: "noop" - -byok_rag: - - rag_id: e2e-test-docs - rag_type: inline::faiss - embedding_model: sentence-transformers/all-mpnet-base-v2 - embedding_dimension: 768 - vector_db_id: ${env.FAISS_VECTOR_STORE_ID} - db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} - score_multiplier: 1.0 - -rag: - inline: - - e2e-test-docs diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-feedback-storage.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-feedback-storage.yaml deleted file mode 100644 index eb6ba2054..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-feedback-storage.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - # Uses a remote llama-stack service - # The instance would have already been started with a llama-stack-run.yaml file - use_as_library_client: false - # Alternative for "as library use" - # use_as_library_client: true - # library_client_config_path: - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/invalid" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -authentication: - module: "noop-with-token" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml deleted file mode 100644 index 3f985b235..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -authentication: - module: "noop" -mcp_servers: - - name: "mcp-file" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3001" - authorization_headers: - Authorization: "/tmp/invalid-mcp-token" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml deleted file mode 100644 index 1a7eb1da5..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -authentication: - module: "noop" -mcp_servers: - - name: "mcp-client" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3001" - authorization_headers: - Authorization: "client" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml deleted file mode 100644 index d73623bc6..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -authentication: - module: "noop" -mcp_servers: - - name: "mcp-file" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3001" - authorization_headers: - Authorization: "/tmp/mcp-secret-token" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml deleted file mode 100644 index 165a1825e..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -authentication: - module: "noop-with-token" -mcp_servers: - - name: "mcp-kubernetes" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3001" - authorization_headers: - Authorization: "kubernetes" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml deleted file mode 100644 index 3241e2a2d..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -authentication: - module: "noop" -mcp_servers: - - name: "mcp-oauth" - provider_id: "model-context-protocol" - url: "http://mock-mcp:3001" - authorization_headers: - Authorization: "oauth" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp.yaml deleted file mode 100644 index cc6ab2e26..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Prow/OpenShift: cluster DNS + MCP mock service (port 3000). Do not use docker names (llama-stack, mock-mcp:3001). -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" -conversation_cache: - type: "sqlite" - sqlite: - db_path: "/tmp/data/conversation-cache.db" -authentication: - module: "noop" -mcp_servers: - - name: "mcp-oauth" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" - authorization_headers: - Authorization: "oauth" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-no-cache.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-no-cache.yaml deleted file mode 100644 index 6c8f31438..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-no-cache.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - # Uses a remote llama-stack service - # The instance would have already been started with a llama-stack-run.yaml file - use_as_library_client: false - # Alternative for "as library use" - # use_as_library_client: true - # library_client_config_path: - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -# NO conversation_cache configured - for testing error handling - -authentication: - module: "noop-with-token" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml deleted file mode 100644 index e2e4bfa16..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: Lightspeed Core Service (RBAC E2E Tests) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: true - workers: 1 - color_log: true - access_log: true - -llama_stack: - use_as_library_client: false - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy - -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -# Conversation cache for storing Q&A history -conversation_cache: - type: "sqlite" - sqlite: - db_path: "/tmp/data/conversation-cache.db" - -# JWK token authentication with role extraction -authentication: - module: "jwk-token" - jwk_config: - url: "http://mock-jwks:8000/.well-known/jwks.json" - jwt_configuration: - user_id_claim: "sub" - username_claim: "name" - # Role rules: extract roles from JWT claims - role_rules: - # Grant 'admin' role to users with admin=true in JWT - - jsonpath: "$.admin" - operator: "equals" - value: [true] - roles: ["admin"] - # Grant 'user' role to users with role=user in JWT - - jsonpath: "$.role" - operator: "equals" - value: ["user"] - roles: ["user"] - # Grant 'viewer' role to users with role=viewer in JWT - - jsonpath: "$.role" - operator: "equals" - value: ["viewer"] - roles: ["viewer"] - # Grant 'query_only' role based on permissions array containing 'query' - - jsonpath: "$.permissions[*]" - operator: "contains" - value: "query" - roles: ["query_only"] - -# Authorization: map roles to actions -authorization: - access_rules: - # Admin role gets full access - - role: "admin" - actions: ["admin"] - # User role can query, access conversations, and provide feedback - - role: "user" - actions: - - "query" - - "streaming_query" - - "get_conversation" - - "list_conversations" - - "delete_conversation" - - "update_conversation" - - "feedback" - - "get_models" - - "get_tools" - - "info" - - "model_override" - # Viewer role can only read (no mutations) - - role: "viewer" - actions: - - "get_conversation" - - "list_conversations" - - "get_models" - - "get_tools" - - "info" - # Query-only role can only query (no model_override - must use defaults) - - role: "query_only" - actions: - - "query" - - "streaming_query" - # Everyone (*) role gets basic info access - - role: "*" - actions: - - "info" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml deleted file mode 100644 index b1fecbdbb..000000000 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - # Uses a remote llama-stack service - # The instance would have already been started with a llama-stack-run.yaml file - use_as_library_client: false - # Alternative for "as library use" - # use_as_library_client: true - # library_client_config_path: - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -authentication: - module: "noop" - -mcp_servers: - # Mock server with client-provided auth - should appear in mcp-auth/client-options response - - name: "github-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" - authorization_headers: - Authorization: "client" - # Mock server with client-provided auth (different header) - should appear in response - - name: "gitlab-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" - authorization_headers: - X-API-Token: "client" - # Mock server with no auth - should NOT appear in response - - name: "public-api" - provider_id: "model-context-protocol" - url: "http://mcp-mock-server:3000" diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 1dce15714..347ca804e 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -141,10 +141,13 @@ log "✅ Mock servers deployed" progress "Deploying lightspeed-stack and llama-stack" # Llama run config: single source with GitHub E2E (tests/e2e/configs/run-ci.yaml). +# Lightspeed stack: same tree as local/docker E2E (tests/e2e/configuration/server-mode). oc create configmap llama-stack-config -n "$NAMESPACE" \ --from-file=run.yaml="$REPO_ROOT/tests/e2e/configs/run-ci.yaml" \ --dry-run=client -o yaml | oc apply -f - -oc create configmap lightspeed-stack-config -n "$NAMESPACE" --from-file=configs/lightspeed-stack.yaml --dry-run=client -o yaml | oc apply -f - +oc create configmap lightspeed-stack-config -n "$NAMESPACE" \ + --from-file=lightspeed-stack.yaml="$REPO_ROOT/tests/e2e/configuration/server-mode/lightspeed-stack.yaml" \ + --dry-run=client -o yaml | oc apply -f - # Create RAG data ConfigMap from the e2e test RAG data log "Creating RAG data ConfigMap..." diff --git a/tests/e2e-prow/rhoai/pipeline.sh b/tests/e2e-prow/rhoai/pipeline.sh index 37d6229b0..dfca0d22f 100755 --- a/tests/e2e-prow/rhoai/pipeline.sh +++ b/tests/e2e-prow/rhoai/pipeline.sh @@ -194,8 +194,11 @@ echo "✅ Mock servers deployed" echo "===== Deploying Services =====" create_secret api-url-secret --from-literal=key="$KSVC_URL" -oc create configmap llama-stack-config -n "$NAMESPACE" --from-file=configs/run.yaml -oc create configmap lightspeed-stack-config -n "$NAMESPACE" --from-file=configs/lightspeed-stack.yaml +# Llama run.yaml: vLLM layout (not under server-mode). Lightspeed: single source with GitHub E2E server-mode. +oc create configmap llama-stack-config -n "$NAMESPACE" \ + --from-file="$REPO_ROOT/tests/e2e-prow/rhoai/configs/run.yaml" +oc create configmap lightspeed-stack-config -n "$NAMESPACE" \ + --from-file="$REPO_ROOT/tests/e2e/configuration/server-mode/lightspeed-stack.yaml" # Create RAG data ConfigMap from the e2e test RAG data echo "Creating RAG data ConfigMap..." diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml index a8bd9f207..49ee71d59 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml @@ -13,7 +13,7 @@ llama_stack: # Alternative for "as library use" # use_as_library_client: true # library_client_config_path: - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml index 1fc83fa4a..e2b468cf0 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml @@ -8,7 +8,7 @@ service: access_log: true llama_stack: use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml index f5d09b2d8..1a2850162 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml @@ -8,7 +8,7 @@ service: access_log: true llama_stack: use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml index 2474cefa4..eb6ba2054 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml @@ -13,7 +13,7 @@ llama_stack: # Alternative for "as library use" # use_as_library_client: true # library_client_config_path: - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml index 90ad70566..6bd1c7b1c 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml index ed6251fcc..7e2798b16 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml index e0f952fc3..8f1218073 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml index a786c041f..493583f12 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml index 999010414..2889d712f 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml index b9125de8e..2f0e1679e 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml index 8c26194e5..36fa111f8 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml index 03ae32ab8..6c8f31438 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml @@ -13,7 +13,7 @@ llama_stack: # Alternative for "as library use" # use_as_library_client: true # library_client_config_path: - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-no-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-no-mcp.yaml index cc699ba89..e7cb24b8b 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-no-mcp.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-no-mcp.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml index 21e4505e2..ea5bce5f3 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml @@ -9,7 +9,7 @@ service: llama_stack: use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml index 1e43005fd..fe8d0891e 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack.yaml @@ -9,7 +9,7 @@ service: llama_stack: # Server mode - connects to separate llama-stack service use_as_library_client: false - url: http://llama-stack:8321 + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 api_key: xyzzy user_data_collection: feedback_enabled: true diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index be94020d3..2d2dd6f2f 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -37,74 +37,29 @@ FALLBACK_MODEL = "gpt-4o-mini" FALLBACK_PROVIDER = "openai" -# Config file mappings: config_name -> (docker_path, prow_path) +# Config file mappings: logical key -> path template (Docker and Prow use the same tree). _CONFIG_PATHS = { - "no-cache": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-no-cache.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-no-cache.yaml", - ), - "auth-noop-token": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-auth-noop-token.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml", - ), - "rbac": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-rbac.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml", - ), - "invalid-feedback-storage": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-feedback-storage.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-feedback-storage.yaml", - ), - "rh-identity": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-auth-rh-identity.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-rh-identity.yaml", - ), + "no-cache": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-no-cache.yaml", + "auth-noop-token": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-auth-noop-token.yaml", + "rbac": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-rbac.yaml", + "invalid-feedback-storage": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-feedback-storage.yaml", + "rh-identity": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-auth-rh-identity.yaml", # Default LCS config for @MCP feature; per-scenario tags select mcp-* variants in before_scenario. - "mcp": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp.yaml", - ), - "mcp-file-auth": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-file-auth.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml", - ), - "invalid-mcp-file-auth": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-mcp-file-auth.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml", - ), - "mcp-kubernetes-auth": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-kubernetes-auth.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml", - ), - "mcp-client-auth": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-client-auth.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml", - ), - "mcp-oauth-auth": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-oauth-auth.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml", - ), - "inline-rag": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-inline-rag.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-inline-rag.yaml", - ), - "mcp-auth": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-auth.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-auth.yaml", - ), - "no-mcp": ( - "tests/e2e/configuration/{mode_dir}/lightspeed-stack-no-mcp.yaml", - "tests/e2e-prow/rhoai/configs/lightspeed-stack-no-mcp.yaml", - ), + "mcp": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp.yaml", + "mcp-file-auth": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-file-auth.yaml", + "invalid-mcp-file-auth": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-mcp-file-auth.yaml", + "mcp-kubernetes-auth": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-kubernetes-auth.yaml", + "mcp-client-auth": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-client-auth.yaml", + "mcp-oauth-auth": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-oauth-auth.yaml", + "inline-rag": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-inline-rag.yaml", + "mcp-auth": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-auth.yaml", + "no-mcp": "tests/e2e/configuration/{mode_dir}/lightspeed-stack-no-mcp.yaml", } def _get_config_path(config_name: str, mode_dir: str) -> str: - """Get the appropriate config path based on environment.""" - docker_path_template, prow_path = _CONFIG_PATHS[config_name] - if is_prow_environment(): - return prow_path - return docker_path_template.format(mode_dir=mode_dir) + """Resolve repo-relative path to a Lightspeed stack YAML for the current mode.""" + return _CONFIG_PATHS[config_name].format(mode_dir=mode_dir) def _fetch_models_from_service() -> dict: diff --git a/tests/e2e/features/inline_rag.feature b/tests/e2e/features/inline_rag.feature index 7f49c9f58..194af71a6 100644 --- a/tests/e2e/features/inline_rag.feature +++ b/tests/e2e/features/inline_rag.feature @@ -5,6 +5,7 @@ Feature: Inline RAG (BYOK) support tests And The system is in default state And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" And The service uses the lightspeed-stack-inline-rag.yaml configuration And The service is restarted diff --git a/tests/e2e/features/steps/common.py b/tests/e2e/features/steps/common.py index de2782cc7..5918e7454 100644 --- a/tests/e2e/features/steps/common.py +++ b/tests/e2e/features/steps/common.py @@ -6,6 +6,7 @@ from behave.runner import Context from tests.e2e.utils.utils import ( + absolute_repo_path, create_config_backup, is_prow_environment, restart_container, @@ -32,7 +33,22 @@ def service_is_started_locally(context: Context) -> None: context.port_llama = os.getenv("E2E_LLAMA_PORT", "8321") -@given("The service uses the {config_name} configuration") # type: ignore[reportCallIssue] +@given('the Lightspeed stack configuration directory is "{directory}"') +def set_lightspeed_stack_config_directory(context: Context, directory: str) -> None: + """Store the repo-relative config folder for ``The service uses the ... configuration``. + + ``configure_service`` joins this path with either + ``/`` when those subdirs exist under ``directory``, + or ```` alone for a flat directory (local Docker and Prow). + + Parameters: + context: Behave context; sets ``lightspeed_stack_config_directory``. + directory: Path relative to the repository root (e.g. ``tests/e2e/configuration``). + """ + context.lightspeed_stack_config_directory = directory.strip().rstrip("/") + + +@given("The service uses the {config_name} configuration") # type: ignore def configure_service(context: Context, config_name: str) -> None: """Switch to the given configuration if not already active. @@ -41,6 +57,12 @@ def configure_service(context: Context, config_name: str) -> None: the same feature are no-ops (detected by backup file existence in Docker or backup key presence in Prow). + Build path from ``lightspeed_stack_config_directory`` (directory step), + defaulting base to ``tests/e2e/configuration`` if that step was omitted; then + ``server-mode`` / ``library-mode`` subdir when present, else flat. On Prow the + resolved path is made absolute from the repo root so ``switch_config`` works + regardless of cwd. + Parameters: ---------- context (Context): Behave context. @@ -50,10 +72,24 @@ def configure_service(context: Context, config_name: str) -> None: return mode_dir = "library-mode" if context.is_library_mode else "server-mode" + raw_base = getattr(context, "lightspeed_stack_config_directory", None) + if raw_base and str(raw_base).strip(): + base = str(raw_base).strip().rstrip("/") + else: + base = "tests/e2e/configuration" + mode_base = os.path.join(base, mode_dir) if is_prow_environment(): - config_path = f"tests/e2e-prow/rhoai/configs/{config_name}" + abs_mode_base = absolute_repo_path(mode_base) + abs_base = absolute_repo_path(base) + if os.path.isdir(abs_mode_base): + config_path = os.path.join(abs_mode_base, config_name) + else: + config_path = os.path.join(abs_base, config_name) else: - config_path = f"tests/e2e/configuration/{mode_dir}/{config_name}" + if os.path.isdir(mode_base): + config_path = os.path.join(mode_base, config_name) + else: + config_path = os.path.join(base, config_name) create_config_backup("lightspeed-stack.yaml") switch_config(config_path) diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 4f92ae10e..65518e305 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -18,12 +18,81 @@ wait_for_pod_health, ) +_DEFAULT_CLUSTER_LIGHTSPEED_CONFIG_DIR = "tests/e2e/configuration/server-mode" + + +def _e2e_repo_root() -> str: + """Absolute path to repository root (for Prow/Konflux runners and local behave). + + Set ``E2E_REPO_ROOT`` when the process cwd is not the repo or checkout layout + differs. Otherwise derived from this file: ``tests/e2e/utils/utils.py`` → root. + """ + env = os.getenv("E2E_REPO_ROOT", "").strip() + if env: + return os.path.abspath(env) + return os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") + ) + + +def absolute_repo_path(repo_relative: str) -> str: + """Return an absolute path for a location given relative to the repository root. + + Used on Prow so ``oc`` and file reads do not depend on the process working + directory. If ``repo_relative`` is already absolute, it is normalized and + returned unchanged. + + Parameters: + repo_relative: Path relative to the repo root, or an absolute path. + + Returns: + Normalized absolute filesystem path. + """ + rel = repo_relative.strip() + if os.path.isabs(rel): + return os.path.normpath(rel) + return os.path.normpath(os.path.join(_e2e_repo_root(), rel)) + def is_prow_environment() -> bool: """Check if running in Prow/OpenShift environment.""" return os.getenv("RUNNING_PROW") is not None +def cluster_lightspeed_config_dir() -> str: + """Directory of Lightspeed YAML files used for Prow/Konflux (ConfigMap sources). + + Paths are relative to the repository root. Pipelines set + ``E2E_LIGHTSPEED_CONFIG_DIR`` explicitly so local and CI agree on one variable; + when unset, defaults to ``tests/e2e/configuration/server-mode``. + + Returns: + Non-empty repo-relative directory path. + """ + raw = os.getenv("E2E_LIGHTSPEED_CONFIG_DIR", _DEFAULT_CLUSTER_LIGHTSPEED_CONFIG_DIR) + stripped = raw.strip() + return stripped if stripped else _DEFAULT_CLUSTER_LIGHTSPEED_CONFIG_DIR + + +def cluster_lightspeed_config_path(filename: str) -> str: + """Absolute path to a Lightspeed stack YAML used for Prow/OpenShift ConfigMap updates. + + The cluster loads config from the ConfigMap; Behave/e2e-ops must read the same + YAML from the **runner filesystem**. Using an absolute path anchored at the repo + root avoids failures when behave's cwd is not the repository root. + + Parameters: + filename: Basename of the Lightspeed stack YAML. + + Returns: + Absolute path suitable for ``switch_config`` and ``oc --from-file`` on CI. + """ + base = cluster_lightspeed_config_dir() + if os.path.isabs(base): + return os.path.normpath(os.path.join(base, filename)) + return os.path.normpath(os.path.join(_e2e_repo_root(), base, filename)) + + def normalize_endpoint(endpoint: str) -> str: """Normalize endpoint to be added into the URL.