more

Signed-off-by: Takeshi Yoneda <[email protected]>

Author: mathetake

cmd/dynamic_module/main.go

@@ -1,3 +1,8 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
 package main
 
 import (
@@ -7,14 +12,15 @@ import (
 	"os"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	otelprom "go.opentelemetry.io/otel/exporters/prometheus"
+
 	"github.com/envoyproxy/ai-gateway/internal/backendauth"
-	"github.com/envoyproxy/ai-gateway/internal/dynamic_module"
-	"github.com/envoyproxy/ai-gateway/internal/dynamic_module/sdk"
+	"github.com/envoyproxy/ai-gateway/internal/dynamicmodule"
+	"github.com/envoyproxy/ai-gateway/internal/dynamicmodule/sdk"
 	"github.com/envoyproxy/ai-gateway/internal/filterapi"
 	"github.com/envoyproxy/ai-gateway/internal/internalapi"
 	"github.com/envoyproxy/ai-gateway/internal/metrics"
-	"github.com/prometheus/client_golang/prometheus"
-	otelprom "go.opentelemetry.io/otel/exporters/prometheus"
 )
 
 func main() {} // This must be present to make a shared library.
@@ -31,7 +37,7 @@ func init() {
 		Level: slog.LevelDebug, // Adjust log level from environment variable if needed.
 	}))
 	if err := filterapi.StartConfigWatcher(context.Background(),
-		os.Getenv("AI_GATEWAY_DYNAMIC_MODULE_FILTER_CONFIG_PATH"), g, logger, time.Second*5); err != nil {
+		os.Getenv("AI_GATEWAY_dynamicmodule_FILTER_CONFIG_PATH"), g, logger, time.Second*5); err != nil {
 		panic("failed to start filter config watcher: " + err.Error())
 	}
 	sdk.NewHTTPFilterConfig = g.newHTTPFilterConfig
@@ -40,7 +46,7 @@ func init() {
 // globalState implements [filterapi.ConfigReceiver] to load filter configuration.
 type globalState struct {
 	fc  *filterapi.RuntimeConfig
-	env *dynamic_module.Env
+	env *dynamicmodule.Env
 }
 
 // newHTTPFilterConfig creates a new http filter based on the config.
@@ -49,9 +55,9 @@ type globalState struct {
 func (g *globalState) newHTTPFilterConfig(name string, _ []byte) sdk.HTTPFilterConfig {
 	switch name {
 	case "ai_gateway.router":
-		return dynamic_module.NewRouterFilterConfig(g.env, &g.fc)
+		return dynamicmodule.NewRouterFilterConfig(g.env, &g.fc)
 	case "ai_gateway.upstream":
-		return dynamic_module.NewUpstreamFilterConfig(g.env)
+		return dynamicmodule.NewUpstreamFilterConfig(g.env)
 	default:
 		panic("unknown filter: " + name)
 	}
@@ -81,21 +87,21 @@ func (g *globalState) initializeEnv() error {
 	}
 
 	endpointPrefixes, err := internalapi.ParseEndpointPrefixes(os.Getenv(
-		"AI_GATEWAY_DYNAMIC_MODULE_FILTER_ENDPOINT_PREFIXES",
+		"AI_GATEWAY_dynamicmodule_FILTER_ENDPOINT_PREFIXES",
 	))
 	if err != nil {
 		return fmt.Errorf("failed to parse endpoint prefixes: %w", err)
 	}
 
 	metricsRequestHeaderAttributes, err := internalapi.ParseRequestHeaderAttributeMapping(os.Getenv(
-		"AI_GATEWAY_DYNAMIC_MODULE_FILTER_METRICS_REQUEST_HEADER_ATTRIBUTES",
+		"AI_GATEWAY_dynamicmodule_FILTER_METRICS_REQUEST_HEADER_ATTRIBUTES",
 	))
 	if err != nil {
 		return fmt.Errorf("failed to parse metrics header mapping: %w", err)
 	}
 
-	g.env = &dynamic_module.Env{
-		RootPrefix:                    os.Getenv("AI_GATEWAY_DYNAMIC_MODULE_ROOT_PREFIX"),
+	g.env = &dynamicmodule.Env{
+		RootPrefix:                    os.Getenv("AI_GATEWAY_dynamicmodule_ROOT_PREFIX"),
 		EndpointPrefixes:              endpointPrefixes,
 		ChatCompletionMetricsFactory:  metrics.NewChatCompletionFactory(meter, metricsRequestHeaderAttributes),
 		MessagesMetricsFactory:        metrics.NewMessagesFactory(meter, metricsRequestHeaderAttributes),

internal/dynamic_module/sdk/cgoflags_darwin.go

@@ -1,4 +1,9 @@
-package dynamic_module
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package dynamicmodule
 
 import (
 	"github.com/envoyproxy/ai-gateway/internal/internalapi"
@@ -21,6 +26,8 @@ const (
 	rerankEndpoint
 	// messagesEndpoint represents the /v1/messages endpoint of anthropic.
 	messagesEndpoint
+	// modelsEndpoint represents the /v1/models endpoint.
+	modelsEndpoint
 )
 
 // Env holds the environment configuration for the dynamic module that is process-wide.

internal/dynamic_module/dynamic_module.go renamed to internal/dynamicmodule/dynamic_module.go

@@ -1,19 +1,26 @@
-package dynamic_module
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package dynamicmodule
 
 import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"path"
 	"strings"
 	"unsafe"
 
+	openaisdk "github.com/openai/openai-go/v2"
+
 	"github.com/envoyproxy/ai-gateway/internal/apischema/anthropic"
 	cohereschema "github.com/envoyproxy/ai-gateway/internal/apischema/cohere"
 	"github.com/envoyproxy/ai-gateway/internal/apischema/openai"
-	"github.com/envoyproxy/ai-gateway/internal/dynamic_module/sdk"
+	"github.com/envoyproxy/ai-gateway/internal/dynamicmodule/sdk"
 	"github.com/envoyproxy/ai-gateway/internal/filterapi"
 	"github.com/envoyproxy/ai-gateway/internal/internalapi"
-	openaisdk "github.com/openai/openai-go/v2"
 )
 
 const routerFilterPointerDynamicMetadataKey = "router_filter_pointer"
@@ -24,11 +31,13 @@ type (
 	// This is mostly for debugging purposes, it does not do anything except
 	// setting a response header with the version of the dynamic module.
 	routerFilterConfig struct {
-		fcr **filterapi.RuntimeConfig
+		fcr              **filterapi.RuntimeConfig
+		prefixToEndpoint map[string]endpoint
 	}
 	// routerFilter implements [sdk.HTTPFilter].
 	routerFilter struct {
-		fc                     *filterapi.RuntimeConfig
+		routerFilterConfig     *routerFilterConfig
+		runtimeFilterConfig    *filterapi.RuntimeConfig
 		endpoint               endpoint
 		originalHeaders        map[string]string
 		originalRequestBody    any
@@ -40,12 +49,24 @@ type (
 
 // NewRouterFilterConfig creates a new instance of an implementation of [sdk.HTTPFilterConfig] for the router filter.
 func NewRouterFilterConfig(env *Env, fcr **filterapi.RuntimeConfig) sdk.HTTPFilterConfig {
-	return &routerFilterConfig{fcr: fcr}
+	prefixToEndpoint := map[string]endpoint{
+		path.Join(env.RootPrefix, env.EndpointPrefixes.OpenAI, "/v1/chat/completions"):   chatCompletionsEndpoint,
+		path.Join(env.RootPrefix, env.EndpointPrefixes.OpenAI, "/v1/completions"):        completionsEndpoint,
+		path.Join(env.RootPrefix, env.EndpointPrefixes.OpenAI, "/v1/embeddings"):         embeddingsEndpoint,
+		path.Join(env.RootPrefix, env.EndpointPrefixes.OpenAI, "/v1/images/generations"): imagesGenerationsEndpoint,
+		path.Join(env.RootPrefix, env.EndpointPrefixes.Cohere, "/v2/rerank"):             rerankEndpoint,
+		path.Join(env.RootPrefix, env.EndpointPrefixes.OpenAI, "/v1/models"):             modelsEndpoint,
+		path.Join(env.RootPrefix, env.EndpointPrefixes.Anthropic, "/v1/messages"):        messagesEndpoint,
+	}
+	return &routerFilterConfig{
+		fcr:              fcr,
+		prefixToEndpoint: prefixToEndpoint,
+	}
 }
 
 // NewFilter implements [sdk.HTTPFilterConfig].
 func (f *routerFilterConfig) NewFilter() sdk.HTTPFilter {
-	return &routerFilter{fc: *f.fcr}
+	return &routerFilter{routerFilterConfig: f, runtimeFilterConfig: *f.fcr}
 }
 
 // RequestHeaders implements [sdk.HTTPFilter].
@@ -55,32 +76,16 @@ func (f *routerFilter) RequestHeaders(e sdk.EnvoyHTTPFilter, _ bool) sdk.Request
 	if queryIndex := strings.Index(p, "?"); queryIndex != -1 {
 		p = p[:queryIndex]
 	}
-	// TODO: prefix config.
-	switch p {
-	case "/v1/chat/completions":
-		f.endpoint = chatCompletionsEndpoint
-		return sdk.RequestHeadersStatusContinue
-	case "/v1/completions":
-		f.endpoint = completionsEndpoint
-		return sdk.RequestHeadersStatusContinue
-	case "/v1/embeddings":
-		f.endpoint = embeddingsEndpoint
-		return sdk.RequestHeadersStatusContinue
-	case "/v1/images/generations":
-		f.endpoint = imagesGenerationsEndpoint
-		return sdk.RequestHeadersStatusContinue
-	case "/cohere/v2/rerank":
-		f.endpoint = rerankEndpoint
-		return sdk.RequestHeadersStatusContinue
-	case "/anthropic/v1/messages":
-		f.endpoint = messagesEndpoint
-		return sdk.RequestHeadersStatusContinue
-	case "/v1/models":
-		return f.handleModelsEndpoint(e)
-	default:
+	ep, ok := f.routerFilterConfig.prefixToEndpoint[p]
+	if !ok {
 		e.SendLocalReply(404, nil, []byte(fmt.Sprintf("unsupported path: %s", p)))
 		return sdk.RequestHeadersStatusStopIteration
 	}
+	f.endpoint = ep
+	if f.endpoint == modelsEndpoint {
+		return f.handleModelsEndpoint(e)
+	}
+	return sdk.RequestHeadersStatusContinue
 }
 
 // RequestBody implements [sdk.HTTPFilter].
@@ -148,7 +153,7 @@ func (f *routerFilter) ResponseBody(sdk.EnvoyHTTPFilter, bool) sdk.ResponseBodyS
 //
 // This is called on request headers phase.
 func (f *routerFilter) handleModelsEndpoint(e sdk.EnvoyHTTPFilter) sdk.RequestHeadersStatus {
-	config := *f.fc
+	config := f.runtimeFilterConfig
 	models := openai.ModelList{
 		Object: "list",
 		Data:   make([]openai.Model, 0, len(config.DeclaredModels)),

internal/dynamic_module/router_filter.go renamed to internal/dynamicmodule/router_filter.go

@@ -1,4 +1,7 @@
-//go:build envoy_v1.36
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
 
 package sdk
 
@@ -175,7 +178,7 @@ func envoy_dynamic_module_on_http_filter_config_new(
 
 //export envoy_dynamic_module_on_http_filter_config_destroy
 func envoy_dynamic_module_on_http_filter_config_destroy(ptr uintptr) {
-	pinnedFilterConfig := unwrapPinnedHTTPFilterConfig(uintptr(ptr))
+	pinnedFilterConfig := unwrapPinnedHTTPFilterConfig(ptr)
 	memManager.unpinHTTPFilterConfig(pinnedFilterConfig)
 }
 
@@ -184,7 +187,7 @@ func envoy_dynamic_module_on_http_filter_new(
 	filterConfigPtr uintptr,
 	_ uintptr,
 ) uintptr {
-	pinnedFilterConfig := unwrapPinnedHTTPFilterConfig(uintptr(filterConfigPtr))
+	pinnedFilterConfig := unwrapPinnedHTTPFilterConfig(filterConfigPtr)
 	filterConfig := pinnedFilterConfig.obj
 	filter := filterConfig.NewFilter()
 	if filter == nil {
@@ -200,7 +203,7 @@ func envoy_dynamic_module_on_http_filter_new(
 func envoy_dynamic_module_on_http_filter_destroy(
 	filterPtr uintptr,
 ) {
-	pinned := unwrapPinnedHTTPFilter(uintptr(filterPtr))
+	pinned := unwrapPinnedHTTPFilter(filterPtr)
 	// Unpin the filter from the memory manager.
 	memManager.unpinHTTPFilter(pinned)
 }
@@ -211,8 +214,8 @@ func envoy_dynamic_module_on_http_filter_request_headers(
 	filterModulePtr uintptr,
 	endOfStream bool,
 ) uintptr {
-	pinned := unwrapPinnedHTTPFilter(uintptr(filterModulePtr))
-	status := pinned.obj.RequestHeaders(envoyFilter{raw: filterEnvoyPtr}, bool(endOfStream))
+	pinned := unwrapPinnedHTTPFilter(filterModulePtr)
+	status := pinned.obj.RequestHeaders(envoyFilter{raw: filterEnvoyPtr}, endOfStream)
 	return uintptr(status)
 }
 
@@ -222,8 +225,8 @@ func envoy_dynamic_module_on_http_filter_request_body(
 	filterModulePtr uintptr,
 	endOfStream bool,
 ) uintptr {
-	pinned := unwrapPinnedHTTPFilter(uintptr(filterModulePtr))
-	status := pinned.obj.RequestBody(envoyFilter{raw: uintptr(filterEnvoyPtr)}, bool(endOfStream))
+	pinned := unwrapPinnedHTTPFilter(filterModulePtr)
+	status := pinned.obj.RequestBody(envoyFilter{raw: filterEnvoyPtr}, endOfStream)
 	return uintptr(status)
 }
 
@@ -238,8 +241,8 @@ func envoy_dynamic_module_on_http_filter_response_headers(
 	filterModulePtr uintptr,
 	endOfStream bool,
 ) uintptr {
-	pinned := unwrapPinnedHTTPFilter(uintptr(filterModulePtr))
-	status := pinned.obj.ResponseHeaders(envoyFilter{raw: uintptr(filterEnvoyPtr)}, bool(endOfStream))
+	pinned := unwrapPinnedHTTPFilter(filterModulePtr)
+	status := pinned.obj.ResponseHeaders(envoyFilter{raw: filterEnvoyPtr}, endOfStream)
 	return uintptr(status)
 }
 
@@ -249,8 +252,8 @@ func envoy_dynamic_module_on_http_filter_response_body(
 	filterModulePtr uintptr,
 	endOfStream bool,
 ) uintptr {
-	pinned := unwrapPinnedHTTPFilter(uintptr(filterModulePtr))
-	status := pinned.obj.ResponseBody(envoyFilter{raw: uintptr(filterEnvoyPtr)}, bool(endOfStream))
+	pinned := unwrapPinnedHTTPFilter(filterModulePtr)
+	status := pinned.obj.ResponseBody(envoyFilter{raw: filterEnvoyPtr}, endOfStream)
 	return uintptr(status)
 }
 
@@ -274,7 +277,14 @@ func envoy_dynamic_module_on_http_filter_http_callout_done(
 	bodyVectorPtr uintptr,
 	bodyVectorSize C.size_t,
 ) {
-	panic("TODO")
+	_ = filterEnvoyPtr
+	_ = filterModulePtr
+	_ = calloutID
+	_ = result
+	_ = headersPtr
+	_ = headersSize
+	_ = bodyVectorPtr
+	_ = bodyVectorSize
 }
 
 //export envoy_dynamic_module_on_http_filter_scheduled
@@ -283,6 +293,9 @@ func envoy_dynamic_module_on_http_filter_scheduled(
 	filterModulePtr uintptr,
 	eventID C.uint64_t,
 ) {
+	_ = filterEnvoyPtr
+	_ = filterModulePtr
+	_ = eventID
 }
 
 // GetRequestHeader implements [EnvoyHTTPFilter].

internal/dynamic_module/sdk/abi_envoy_v1.36.go renamed to internal/dynamicmodule/sdk/abi_envoy_v1.36.go

@@ -0,0 +1,11 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build darwin && cgo
+
+package sdk
+
+// #cgo LDFLAGS: -Wl,-undefined,dynamic_lookup
+import "C"

internal/dynamicmodule/sdk/cgoflags_darwin.go

@@ -1,3 +1,8 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
 package sdk
 
 import "io"

internal/dynamic_module/sdk/gosdk.go renamed to internal/dynamicmodule/sdk/gosdk.go

@@ -1,3 +1,8 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
 package sdk
 
 import (