Merge branch 'issue_241_new' of https://github.com/ericmariasis/ai-ga…

…teway into issue_241_new merge
envoyproxy · Feb 7, 2025 · b67065a · b67065a
2 parents cb3d2d6 + ddb9858
commit b67065a
Show file tree

Hide file tree

Showing 66 changed files with 525 additions and 313 deletions.
diff --git a/.github/workflows/issue_title_check.yaml b/.github/workflows/issue_title_check.yaml
@@ -0,0 +1,36 @@
+name: Issue Title Check
+
+on:
+  issues:
+    types: [opened, edited]
+
+permissions:
+  issues: write
+
+jobs:
+  check_title_length:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Issue Title Length
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const issueNumber = context.issue.number;
+            const repoOwner = context.repo.owner;
+            const repoName = context.repo.repo;
+
+            const issueTitle = context.payload.issue.title;
+            const length = issueTitle.length;
+
+            console.log(`Title: ${issueTitle}`);
+            console.log(`Length: ${length}`);
+
+            if (length > 80) {
+              await github.rest.issues.createComment({
+                owner: repoOwner,
+                repo: repoName,
+                issue_number: issueNumber,
+                body: "The issue title is too long (over 80 characters). Please shorten it and ensure it is well summarized."
+              });
+            }
diff --git a/.github/workflows/pr_style_check.yaml b/.github/workflows/pr_style_check.yaml
@@ -17,6 +17,7 @@ jobs:
     env:
       # Do not use ${{ github.event.pull_request.body }} directly in run command.
       BODY: ${{ github.event.pull_request.body }}
+    if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Check comment out lines
         run: |
@@ -41,6 +42,7 @@ jobs:
   title:
     name: Title
     runs-on: ubuntu-latest
+    if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - uses: amannn/action-semantic-pull-request@v5
         env:
@@ -73,11 +75,11 @@ jobs:
         env:
           # Do not use ${{ github.event.pull_request.title }} directly in run command.
           TITLE: ${{ github.event.pull_request.title }}
-        # We want to make sure that each commit "subject" is under 60 characters not to
+        # We want to make sure that each commit "subject" is <=60 characters not to
         # be truncated in the git log as well as in the GitHub UI.
         # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/process/submitting-patches.rst?id=bc7938deaca7f474918c41a0372a410049bd4e13#n664
         run: |
-          if (( ${#TITLE} >= 60 )); then
-            echo "The PR title is too long. Please keep it under 60 characters."
+          if (( ${#TITLE} > 60 )); then
+            echo "The PR title is too long. Please keep it <=60 characters."
             exit 1
           fi
diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml
@@ -7,6 +7,9 @@ on:
     branches:
       - main
 
+permissions:
+  contents: read
+
 jobs:
   style:
     name: Check

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -29,6 +29,10 @@ concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}-${{ github.actor }}-${{ github.event_name }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   unittest:
     if: github.event_name == 'pull_request' || github.event_name == 'push'
@@ -47,12 +51,11 @@ jobs:
             ~/go/pkg/mod
             ~/go/bin
           key: unittest-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }}
-      - name: Run unit tests
-        run: make test-coverage
+      - run: make test-coverage
 
-  test_cel_validation:
+  test_crdcel:
     if: github.event_name == 'pull_request' || github.event_name == 'push'
-    name: CEL Validation Test
+    name: CRD CEL Validation Test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -67,8 +70,7 @@ jobs:
             ~/go/pkg/mod
             ~/go/bin
           key: celvalidation-test-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }}
-      - name: Run unit tests
-        run: make test-cel
+      - run: make test-crdcel
 
   test_controller:
     if: github.event_name == 'pull_request' || github.event_name == 'push'
@@ -87,8 +89,7 @@ jobs:
             ~/go/pkg/mod
             ~/go/bin
           key: controller-test-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }}
-      - name: Run unit tests
-        run: make test-controller
+      - run: make test-controller
 
   test_extproc:
     name: External Processor Test (Envoy ${{ matrix.name }})
@@ -129,8 +130,7 @@ jobs:
           docker run -v $ENVOY_BIN_DIR:/tmp/ci -w /tmp/ci \
           --entrypoint /bin/cp ${{ matrix.envoy_version }} /usr/local/bin/envoy .
           echo $ENVOY_BIN_DIR >> $GITHUB_PATH
-      - name: Run unit tests
-        env:
+      - env:
           TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BEDROCK_USER_AWS_ACCESS_KEY_ID }}
           TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_USER_AWS_SECRET_ACCESS_KEY }}
           TEST_OPENAI_API_KEY: ${{ secrets.ENVOY_AI_GATEWAY_OPENAI_API_KEY }}
@@ -170,8 +170,7 @@ jobs:
             ~/go/bin
           key: e2e-test-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }}
       - uses: docker/setup-buildx-action@v3
-      - name: Run E2E tests
-        env:
+      - env:
           EG_VERSION: ${{ matrix.envoy_gateway_version }}
           TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_BEDROCK_USER_AWS_ACCESS_KEY_ID }}
           TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_BEDROCK_USER_AWS_SECRET_ACCESS_KEY }}
@@ -182,7 +181,7 @@ jobs:
     # Docker builds are verified in test_e2e job, so we only need to push the images when the event is a push event.
     if: github.event_name == 'push'
     name: Push Docker Images
-    needs: [unittest, test_cel_validation, test_controller, test_extproc, test_e2e]
+    needs: [unittest, test_crdcel, test_controller, test_extproc, test_e2e]
     uses: ./.github/workflows/docker_builds_template.yaml
 
   helm_push:
@@ -199,6 +198,4 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Push Helm chart
-        run: |
-          make helm-push
+      - run: make helm-push
diff --git a/.golangci.yml b/.golangci.yml
@@ -71,13 +71,7 @@ linters-settings:
   govet:
     enable-all: true
     disable:
-      - shadow
       - fieldalignment
-  revive:
-    rules:
-      # TODO: enable if-return check
-      - name: if-return
-        disabled: true
   testifylint:
     disable:
       - float-compare

diff --git a/Makefile b/Makefile
@@ -32,12 +32,12 @@ help:
 	@echo "All core targets needed for contributing:"
 	@echo "  precommit       	 Run all necessary steps to prepare for a commit."
 	@echo "  test            	 Run the unit tests for the codebase."
-	@echo "  test-coverage     	 Run the unit tests for the codebase with coverage check."
-	@echo "  test-cel        	 Run the integration tests of CEL validation rules in API definitions with envtest."
+	@echo "  test-coverage		 Run the unit tests for the codebase with coverage check."
+	@echo "  test-crdcel      	 Run the integration tests of CEL validation in CRD definitions with envtest."
 	@echo "                  	 This will be needed when changing API definitions."
 	@echo "  test-extproc    	 Run the integration tests for extproc without controller or k8s at all."
 	@echo "  test-controller	 Run the integration tests for the controller with envtest."
-	@echo "  test-e2e          	 Run the end-to-end tests with a local kind cluster."
+	@echo "  test-e2e       	 Run the end-to-end tests with a local kind cluster."
 	@echo ""
 	@echo "For example, 'make precommit test' should be enough for initial iterations, and later 'make test-cel' etc. for the normal development cycle."
 	@echo "Note that some cases run by test-e2e or test-extproc use credentials and these will be skipped when not available."
@@ -48,7 +48,7 @@ help:
 .PHONY: lint
 lint: golangci-lint
 	@echo "lint => ./..."
-	@$(GOLANGCI_LINT) run --build-tags==test_cel_validation,test_controller,test_extproc ./...
+	@$(GOLANGCI_LINT) run --build-tags==test_crdcel,test_controller,test_extproc ./...
 
 .PHONY: codespell
 CODESPELL_SKIP := $(shell cat .codespell.skip | tr \\n ',')
@@ -124,15 +124,15 @@ test:
 
 ENVTEST_K8S_VERSIONS ?= 1.29.0 1.30.0 1.31.0
 
-# This runs the integration tests of CEL validation rules in API definitions.
+# This runs the integration tests of CEL validation rules in CRD definitions.
 #
 # This requires the EnvTest binary to be built.
-.PHONY: test-cel
-test-cel: envtest apigen
+.PHONY: test-crdcel
+test-crdcel: envtest apigen
 	@for k8sVersion in $(ENVTEST_K8S_VERSIONS); do \
   		echo "Run CEL Validation on k8s $$k8sVersion"; \
         KUBEBUILDER_ASSETS="$$($(ENVTEST) use $$k8sVersion -p path)" \
-                 go test ./tests/cel-validation $(GO_TEST_ARGS) $(GO_TEST_E2E_ARGS) --tags test_cel_validation; \
+                 go test ./tests/crdcel $(GO_TEST_ARGS) $(GO_TEST_E2E_ARGS) --tags test_crdcel; \
     done
 
 # This runs the end-to-end tests for extproc without controller or k8s at all.

diff --git a/Makefile.tools.mk b/Makefile.tools.mk
@@ -16,15 +16,15 @@ CRD_REF_DOCS = $(LOCALBIN)/crd-ref-docs
 GO_TEST_COVERAGE ?= $(LOCALBIN)/go-test-coverage
 
 ## Tool versions.
-CONTROLLER_TOOLS_VERSION ?= v0.17.1
-ENVTEST_VERSION ?= release-0.19
-GOLANGCI_LINT_VERSION ?= v1.63.4
-GO_FUMPT_VERSION ?= v0.7.0
-GCI_VERSION ?= v0.13.5
-EDITORCONFIG_CHECKER_VERSION ?= v3.1.2
-KIND_VERSION ?= v0.26.0
-CRD_REF_DOCS_VERSION ?= v0.1.0
-GO_TEST_COVERAGE_VERSION ?= v2.11.4
+CONTROLLER_TOOLS_VERSION ?= v0.17.1 # https://github.com/kubernetes-sigs/controller-tools/releases
+ENVTEST_VERSION ?= release-0.20 # https://github.com/kubernetes-sigs/controller-runtime/releases Note: this needs to point to a release branch.
+GOLANGCI_LINT_VERSION ?= v1.63.4 # https://github.com/golangci/golangci-lint/releases
+GO_FUMPT_VERSION ?= v0.7.0 # https://github.com/mvdan/gofumpt/releases
+GCI_VERSION ?= v0.13.5 # https://github.com/daixiang0/gci/releases
+EDITORCONFIG_CHECKER_VERSION ?= v3.2.0 # https://github.com/editorconfig-checker/editorconfig-checker/releases
+KIND_VERSION ?= v0.26.0 # https://github.com/kubernetes-sigs/kind/releases
+CRD_REF_DOCS_VERSION ?= v0.1.0 # https://github.com/elastic/crd-ref-docs/releases
+GO_TEST_COVERAGE_VERSION ?= v2.11.4 # https://github.com/vladopajic/go-test-coverage/releases
 
 .PHONY: golangci-lint
 golangci-lint: $(GOLANGCI_LINT)

diff --git a/api/v1alpha1/api.go b/api/v1alpha1/api.go
@@ -20,8 +20,21 @@ import (
 // on the output schema of the AIServiceBackend while doing the other necessary jobs like
 // upstream authentication, rate limit, etc.
 //
-// AIGatewayRoute generates a HTTPRoute resource based on the configuration basis for routing the traffic.
-// The generated HTTPRoute has the owner reference set to this AIGatewayRoute.
+// For Advanced Users: Envoy AI Gateway will generate the following k8s resources corresponding to the AIGatewayRoute:
+//
+//   - Deployment, Service, and ConfigMap of the k8s API for the AI Gateway filter.
+//     The name of these resources are `ai-eg-route-extproc-${name}`.
+//   - HTTPRoute of the Gateway API as a top-level resource to bind all backends.
+//     The name of the HTTPRoute is the same as the AIGatewayRoute.
+//   - EnvoyExtensionPolicy of the Envoy Gateway API to attach the AI Gateway filter into the HTTPRoute.
+//     The name of the EnvoyExtensionPolicy is `ai-eg-route-extproc-${name}` which is the same as the Deployment, etc.
+//   - HTTPRouteFilter of the Envoy Gateway API per namespace for automatic hostname rewrite.
+//     The name of the HTTPRouteFilter is `ai-eg-host-rewrite`.
+//
+// All of these resources are created in the same namespace as the AIGatewayRoute. Note that this is the implementation
+// detail subject to change. If you want to customize the default behavior of the Envoy AI Gateway, you can use these
+// resources as a reference and create your own resources. Alternatively, you can use EnvoyPatchPolicy API of the Envoy
+// Gateway to patch the generated resources. For example, you can insert a custom filter into the filter chain.
 type AIGatewayRoute struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

diff --git a/examples/token_ratelimit/token_ratelimit.yaml b/examples/token_ratelimit/token_ratelimit.yaml
@@ -37,7 +37,6 @@ spec:
               value: gpt-4o-mini
       backendRefs:
         - name: envoy-ai-gateway-token-ratelimit-testupstream
-          weight: 100
   # The following metadata keys are used to store the costs from the LLM request.
   llmRequestCosts:
     - metadataKey: llm_input_token
@@ -46,6 +45,12 @@ spec:
       type: OutputToken
     - metadataKey: llm_total_token
       type: TotalToken
+    # This configures the token limit based on the CEL expression.
+    # For a demonstration purpose, the CEL expression returns 100000000 only when the input token is 3,
+    # otherwise it returns 0 (no token usage).
+    - metadataKey: llm_cel_calculated_token
+      type: CEL
+      celExpression: "input_tokens == uint(3) ? 100000000 : 0"
 ---
 apiVersion: aigateway.envoyproxy.io/v1alpha1
 kind: AIServiceBackend
@@ -139,3 +144,22 @@ spec:
               metadata:
                 namespace: io.envoy.ai_gateway
                 key: llm_total_token
+
+        # Repeat the same configuration for a different token type.
+        # This configures the token limit based on the CEL expression.
+        - clientSelectors:
+            - headers:
+                - name: x-user-id
+                  type: Distinct
+          limit:
+            requests: 1000
+            unit: Hour
+          cost:
+            request:
+              from: Number
+              number: 0
+            response:
+              from: Metadata
+              metadata:
+                namespace: io.envoy.ai_gateway
+                key: llm_cel_calculated_token
diff --git a/filterapi/filterconfig_test.go b/filterapi/filterconfig_test.go
@@ -1,6 +1,7 @@
 package filterapi_test
 
 import (
+	"context"
 	"log/slog"
 	"os"
 	"path"
@@ -22,7 +23,7 @@ func TestDefaultConfig(t *testing.T) {
 	err = yaml.Unmarshal([]byte(filterapi.DefaultConfig), &cfg)
 	require.NoError(t, err)
 
-	err = server.LoadConfig(&cfg)
+	err = server.LoadConfig(context.Background(), &cfg)
 	require.NoError(t, err)
 }
 

diff --git a/go.mod b/go.mod
@@ -3,15 +3,15 @@ module github.com/envoyproxy/ai-gateway
 go 1.23.5
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.35.0
+	github.com/aws/aws-sdk-go-v2 v1.36.0
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8
-	github.com/aws/aws-sdk-go-v2/config v1.29.3
+	github.com/aws/aws-sdk-go-v2/config v1.29.4
 	github.com/envoyproxy/gateway v1.3.0
-	github.com/envoyproxy/go-control-plane/envoy v1.32.3
+	github.com/envoyproxy/go-control-plane/envoy v1.32.4
 	github.com/go-logr/logr v1.4.2
-	github.com/google/cel-go v0.23.1
+	github.com/google/cel-go v0.23.2
 	github.com/google/go-cmp v0.6.0
-	github.com/openai/openai-go v0.1.0-alpha.50
+	github.com/openai/openai-go v0.1.0-alpha.51
 	github.com/stretchr/testify v1.10.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.27.0
@@ -32,16 +32,16 @@ require (
 require (
 	cel.dev/expr v0.19.2 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.56 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.26 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.30 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.30 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.57 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.12 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
 	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect