diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 14d0c1681d..e1441aaa67 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -96,6 +96,10 @@ jobs: run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + # TestActorEgress in the networking suite builds its Actor from the + # ate-demo-egress/egress ActorTemplate, so the fixture has to exist before. + run: hack/install-ate-kind.sh --deploy-demo-egress - name: Wait for micro-VM golden snapshot run: | kubectl --context kind-kind wait --for=condition=Ready \ diff --git a/Makefile b/Makefile index 3eea157f4d..dc788d3ac3 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ build-atenet: .PHONY: build-demos build-demos: - $(KO) build --ldflags="$(LDFLAGS)" ./demos/counter + $(KO) build --ldflags="$(LDFLAGS)" ./demos/counter ./demos/egress .PHONY: test test: diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 8045f4f237..d18b7c3666 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -7,7 +7,7 @@ Router has several responsibilities: Services in Kubernetes. With `--atenet-router=agentgateway`, the sidecar uses a static ConfigMap and atenet does not start an xDS server. -* ext_proc server for the proxy. To make the deployment and debugging easier, we will run this component together +* ext_proc server for the dataplane. To make the deployment and debugging easier, we will run this component together with the router, but this will be split later into its own component. * ext_proc will call into the ATE gRPC API to get the set of relevant backends (specific the worker IP) and route the traffic accordingly @@ -20,12 +20,60 @@ Router has several responsibilities: bounded wait elapses, instead of failing fast. See [docs/request-parking.md](../../../../../docs/request-parking.md). * Drains gracefully on SIGTERM: flips `/readyz` so the Service stops sending - new connections, waits out endpoint propagation (`--drain-delay`), drives - Envoy's admin API to drain established connections, gracefully stops the - ext_proc server so parked requests finish normally (`--drain-timeout`, - derived from the parking budget), then writes a drain-complete marker that - releases the Envoy container's `preStop` hook. See `drain.go` and - `envoydrain.go`. + new connections, waits out endpoint propagation (`--drain-delay`), drains the + dataplane's established connections (Envoy only — driven over its admin API; + agentgateway manages its own termination), gracefully stops the ext_proc + server so parked requests finish normally (`--drain-timeout`, derived from + the parking budget), then writes a drain-complete marker that releases the + dataplane container's `preStop` hook. See `drain.go` and `envoydrain.go`. +* Authenticates actor identity on egress: on every CONNECT, the egress + gateway's ext_proc handler re-verifies the actor's client certificate against + the actor-identity CA, reads the `ActorIdentity` X.509 extension out of it, + and checks the certified UID against the ATE API. + +## packages + +The ext_proc server handles both traffic directions, and they apply opposite +trust models — egress derives the actor identity from a client certificate the +gateway verified against the actor-identity CA, ingress treats every request +header as unauthenticated client input — so the two are kept in separate +packages that cannot reach into each other: + +* `extproc` — the mux, and nothing else. It terminates the ext_proc stream, + decides which direction a request arrived on, dispatches to the `Handler` + registered for that direction, and records latency and outcome. It also owns + the vocabulary both handlers share (`RequestMetadata`, `Result`, `ReqError`). + It imports neither handler package. +* `ingress` — resume, park, and route to the actor's worker. +* `egress` — certificate-based actor-identity authentication for outbound + CONNECTs. + +Direction is decided by the filter chain the dataplane says accepted the +request (`xds.filter_chain_name`, an Envoy attribute the egress gateway is +configured to send), never by anything in the request itself, so a client +cannot pick the egress path by crafting one. `router` itself does the wiring. + +## modes + +One binary serves both directions. `--mode` selects which: + +| `--mode` | ext_proc handlers | xDS server + ActorTemplate controller | Kubernetes access | +| --- | --- | --- | --- | +| `ingress` | ingress | yes | yes | +| `egress` | egress | no | none | +| `all` (default) | both | yes | yes | + +The mux refuses a direction this instance was not started to serve (404) rather +than falling back to the other handler, which would run the request through the +wrong trust model. + +Ingress and egress are deployed separately today — `atenet-router` fronts the +ingress dataplane, `atenet-egress` the egress gateway — because the two scale +independently, not because they need separate binaries. + +The `--atenet-router` choice only applies to the ingress dataplane. The egress +gateway is its own Deployment with a statically configured Envoy, so +`--atenet-router=agentgateway` leaves it untouched. ## status page diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 4c199aab1d..018fb18d01 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -20,6 +20,8 @@ import ( "time" "github.com/spf13/cobra" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func NewRouterCmd() *cobra.Command { @@ -27,7 +29,7 @@ func NewRouterCmd() *cobra.Command { cmd := &cobra.Command{ Use: "router", - Short: "Router components including xDS server and Envoy ExtProc gateway processing server", + Short: "Router components including the Envoy xDS server and the ext_proc gateway processing server", RunE: func(cmd *cobra.Command, args []string) error { srv, err := NewRouterServer(cfg) if err != nil { @@ -39,6 +41,7 @@ func NewRouterCmd() *cobra.Command { }, } + cmd.Flags().StringVar((*string)(&cfg.Mode), "mode", string(ModeAll), fmt.Sprintf("Traffic direction this instance serves: %q (also runs the ingress control plane — the xDS server and ActorTemplate controller — for an Envoy dataplane), %q (ext_proc only, needs no Kubernetes access), or %q for both. The ext_proc mux refuses a direction this instance was not started to serve rather than falling back to the other one", ModeIngress, ModeEgress, ModeAll)) cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error") cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().BoolVar(&cfg.Standalone, "standalone", false, "Run in standalone mode, bypassing creation of managed deployment and services in Kubernetes cluster") @@ -48,17 +51,18 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections") - cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server") - cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the Envoy External Processing (ext_proc) server") + cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the External Processing (ext_proc) server the dataplane calls") + cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the External Processing (ext_proc) server") cmd.Flags().StringVar(&cfg.EnvoyImage, "envoy-image", "envoyproxy/envoy:v1.30-latest", "Image URI used for dynamically launched router instances") cmd.Flags().StringVar(&cfg.TemplatesFile, "actor-templates-file", "", "Path to offline YAML configuration file listing ActorTemplates") cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)") cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services") - cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") + cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the router dataplane") cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") cmd.Flags().StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80).") cmd.Flags().StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") cmd.Flags().StringVar(&cfg.UpstreamSpiffePrefix, "upstream-spiffe-prefix", "spiffe://cluster.local/", "SPIFFE URI SAN prefix (trust domain) the actor's atunnel server cert must match. Empty falls back to default SAN check against the dialed pod IP (which SPIFFE-only certs never match).") + cmd.Flags().StringVar(&cfg.ActorIdentityCAFile, "actor-identity-ca-file", "", "PEM trust bundle for the actor-identity CA, used to verify the actor client certificates presented on egress CONNECTs. Required by the egress gateway's ext_proc sidecar; empty (the default) leaves egress authentication unconfigured and every egress CONNECT is denied.") // Envoy learns the collector over xDS rather than from its own environment, // so the router has to carry the address for it. Defaulting to // OTEL_EXPORTER_OTLP_ENDPOINT — the same variable the router's own exporter @@ -70,11 +74,11 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().BoolVar(&cfg.Auth.AteapiUseTokenAuth, "ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") cmd.Flags().DurationVar(&cfg.RouteTimeout, "route-timeout", defaultRouteTimeout, "Envoy's end-to-end timeout on the workload route, bounding one request from the ingress listener to the actor's response. Raise it for actors whose turns legitimately run long — a harness relaying an LLM completion holds the request open for the whole generation. This does not cover the resume that may precede the request; see --parked-request-budget") - cmd.Flags().DurationVar(&cfg.ParkedRequest.Budget, "parked-request-budget", defaultParkedRequestBudget, "Maximum time a resume flight keeps a request parked (held and retried) waiting for its actor to become routable; concurrent requests for the same actor share one flight and its budget") - cmd.Flags().IntVar(&cfg.ParkedRequest.Max, "parked-request-max", defaultParkedRequestMax, "Maximum number of requests that may be parked simultaneously; excess requests are shed with 503. 0 disables parking (requests fail fast on worker-pool saturation)") - cmd.Flags().DurationVar(&cfg.ParkedRequest.RetryInterval, "parked-request-retry-interval", defaultParkedRequestRetryInterval, "Delay before a parked request's first resume retry") - cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryFactor, "parked-request-retry-factor", defaultParkedRequestRetryFactor, "Multiplier applied to the retry delay after each attempt; must be >= 1") - cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryJitter, "parked-request-retry-jitter", defaultParkedRequestRetryJitter, "Random fraction in [0, 1) added to each retry delay to de-synchronize parked requests") + cmd.Flags().DurationVar(&cfg.ParkedRequest.Budget, "parked-request-budget", ingress.DefaultParkedRequestBudget, "Maximum time a resume flight keeps a request parked (held and retried) waiting for its actor to become routable; concurrent requests for the same actor share one flight and its budget") + cmd.Flags().IntVar(&cfg.ParkedRequest.Max, "parked-request-max", ingress.DefaultParkedRequestMax, "Maximum number of requests that may be parked simultaneously; excess requests are shed with 503. 0 disables parking (requests fail fast on worker-pool saturation)") + cmd.Flags().DurationVar(&cfg.ParkedRequest.RetryInterval, "parked-request-retry-interval", ingress.DefaultParkedRequestRetryInterval, "Delay before a parked request's first resume retry") + cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryFactor, "parked-request-retry-factor", ingress.DefaultParkedRequestRetryFactor, "Multiplier applied to the retry delay after each attempt; must be >= 1") + cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryJitter, "parked-request-retry-jitter", ingress.DefaultParkedRequestRetryJitter, "Random fraction in [0, 1) added to each retry delay to de-synchronize parked requests") cmd.Flags().IntVar(&cfg.ExtProcMaxRequests, "extproc-max-requests", 0, "Circuit-breaker max_requests for Envoy's ext_proc cluster; 0 (the default) derives it as twice --parked-request-max (minimum 1024). Explicit values must be >= --parked-request-max: every parked request holds one slot for its full wait, and the excess is fast-path headroom") // Graceful shutdown knobs. The router sits behind a Service, so // route-drain window is needed: after SIGTERM the readiness flip @@ -82,7 +86,7 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().DurationVar(&cfg.DrainDelay, "drain-delay", 13*time.Second, "How long to keep serving after SIGTERM before starting the drain, covering readiness-probe detection and Service endpoint propagation") cmd.Flags().DurationVar(&cfg.DrainTimeout, "drain-timeout", 0, "Deadline for the ext_proc drain on shutdown; streams still open past it (parked requests included) are forcefully cancelled. 0 (the default) derives --parked-request-budget + the actor route timeout + margin so parked requests always finish normally. Explicit values must be >= --parked-request-budget") cmd.Flags().StringVar(&cfg.EnvoyAdminAddr, "envoy-admin-address", "127.0.0.1:9901", "Envoy admin interface the shutdown sequence drives to drain the sidecar (healthcheck/fail, drain_listeners, stats polling). Ignored with --atenet-router=agentgateway") - cmd.Flags().StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the Envoy container's preStop hook polls for it so Envoy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake") + cmd.Flags().StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the dataplane container's preStop hook polls for it so the proxy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake") return cmd } diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 833ea46304..40876ab398 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -17,6 +17,8 @@ package router import ( "fmt" "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) type atenetRouter string @@ -26,6 +28,43 @@ const ( atenetRouterAgentgateway atenetRouter = "agentgateway" ) +// Mode selects which ext_proc directions an atenet instance serves. One binary +// implements both, as two handlers behind the same ext_proc mux, but a +// deployment usually fronts one dataplane proxy and only needs the matching +// direction: ingress and egress scale independently, so they run as separate +// Deployments (atenet-router and atenet-egress). +// +// The mode is a floor on what an instance will answer, not just a hint. The mux +// refuses a direction it has no handler for rather than falling back to the +// other one, and an egress-only instance also skips the ingress control plane +// (the xDS server and the ActorTemplate controller), which is what lets it run +// without any Kubernetes access at all. +type Mode string + +const ( + // ModeIngress serves actor-addressed traffic arriving at the ingress + // gateway, and runs the ingress control plane — the xDS server and + // ActorTemplate controller — that configures its dataplane. Only the Envoy + // dataplane takes configuration from it; agentgateway is statically + // configured. + ModeIngress Mode = "ingress" + // ModeEgress serves actor CONNECTs leaving through the egress gateway. + // Nothing else runs: the egress gateway is statically configured, so there + // is no xDS server, no ActorTemplate controller, and no Kubernetes client. + ModeEgress Mode = "egress" + // ModeAll serves both directions from one instance. This is the default, + // and what a single-gateway or local development setup wants. + ModeAll Mode = "all" +) + +// ServesIngress reports whether this mode answers ingress requests. It also +// gates the ingress control plane: the xDS server that configures the ingress +// dataplane (Envoy only) and the ActorTemplate controller that feeds it. +func (m Mode) ServesIngress() bool { return m != ModeEgress } + +// ServesEgress reports whether this mode answers egress CONNECTs. +func (m Mode) ServesEgress() bool { return m != ModeIngress } + // authConfig holds the router's client-auth settings for dialing ateapi. // AteapiCAFile always verifies ateapi's serving cert (the servicedns trust // bundle in-cluster). By default the router presents AteapiClientCertPath @@ -42,6 +81,8 @@ type authConfig struct { // routerConfig holds deployment setup and endpoint options for the router node instance. type routerConfig struct { + // Mode restricts the instance to one traffic direction. Empty means ModeAll. + Mode Mode Standalone bool AtenetRouter string Namespace string @@ -67,8 +108,17 @@ type routerConfig struct { // UpstreamSpiffePrefix validates the actor's atunnel server cert by its // SPIFFE URI SAN prefix (trust domain) instead of the dialed pod IP. UpstreamSpiffePrefix string - LogLevel string - MetricsAddr string + + // ActorIdentityCAFile is the PEM trust bundle for the actor-identity CA, + // used by the egress gateway's ext_proc sidecar to verify the actor client + // certificates atunnel presents on egress CONNECTs. Only that deployment + // sets it; empty leaves egress authentication unconfigured, which makes + // every egress CONNECT fail closed and is correct for an ingress-only + // router (it never sees the egress listener). + ActorIdentityCAFile string + + LogLevel string + MetricsAddr string // OtlpCollectorAddress is the OTLP gRPC collector that Envoy reports // tracing spans to, as host:port or an http:// URL. It defaults to // OTEL_EXPORTER_OTLP_ENDPOINT — Envoy gets its whole configuration over @@ -91,7 +141,8 @@ type routerConfig struct { // ParkedRequest configures request parking: hold and retry requests whose // actor cannot be served immediately due to transient worker-pool // saturation, instead of failing fast. A non-positive Max disables parking. - ParkedRequest ParkedRequestConfig + // Ingress-only: egress never resumes an actor. + ParkedRequest ingress.ParkedRequestConfig // ExtProcMaxRequests is the circuit-breaker max_requests Envoy applies to // the ext_proc cluster. Every parked request holds one slot for its entire @@ -111,8 +162,8 @@ type routerConfig struct { EnvoyAdminAddr string // DrainCompleteFile is the marker file written once shutdown completes, - // releasing Envoy's preStop hook on the shared emptyDir. Removed at startup to - // defuse stale markers. Empty disables the handshake. + // releasing the dataplane container's preStop hook on the shared emptyDir. + // Removed at startup to defuse stale markers. Empty disables the handshake. DrainCompleteFile string } @@ -154,7 +205,7 @@ const drainTimeoutMargin = 5 * time.Second // route ceiling cannot silently stretch shutdown past the pod's grace period // (see defaultRouteTimeout); operators pair a long route timeout with an // explicit --drain-timeout instead. -func (c routerConfig) drainTimeout(parkCfg ParkedRequestConfig) time.Duration { +func (c routerConfig) drainTimeout(parkCfg ingress.ParkedRequestConfig) time.Duration { if c.DrainTimeout > 0 { return c.DrainTimeout } @@ -169,7 +220,12 @@ func (c routerConfig) validate() error { default: return fmt.Errorf("--atenet-router must be %q or %q, got %q", atenetRouterEnvoy, atenetRouterAgentgateway, c.AtenetRouter) } - if err := c.ParkedRequest.validate(); err != nil { + switch c.Mode { + case "", ModeIngress, ModeEgress, ModeAll: + default: + return fmt.Errorf("--mode must be one of %q, %q, or %q, got %q", ModeIngress, ModeEgress, ModeAll, c.Mode) + } + if err := c.ParkedRequest.Validate(); err != nil { return err } @@ -186,9 +242,9 @@ func (c routerConfig) validate() error { if c.DrainTimeout < 0 { return fmt.Errorf("--drain-timeout must not be negative, got %s (0 derives it from --parked-request-budget)", c.DrainTimeout) } - if c.DrainTimeout > 0 && c.ParkedRequest.enabled() && c.DrainTimeout < c.ParkedRequest.normalized().Budget { + if c.DrainTimeout > 0 && c.ParkedRequest.Enabled() && c.DrainTimeout < c.ParkedRequest.Normalized().Budget { return fmt.Errorf("--drain-timeout (%s) must be >= --parked-request-budget (%s): a drain shorter than the parking budget resets parked requests on shutdown instead of letting them finish", - c.DrainTimeout, c.ParkedRequest.normalized().Budget) + c.DrainTimeout, c.ParkedRequest.Normalized().Budget) } return nil } diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index 2dfeae9664..4e41060dd2 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -18,6 +18,8 @@ import ( "strings" "testing" "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func TestRouterConfigValidate(t *testing.T) { @@ -27,16 +29,16 @@ func TestRouterConfigValidate(t *testing.T) { wantErr string // substring; empty means valid }{ { - name: "atenet-router defaults to envoy", - cfg: routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, + name: "defaults are valid (auto breaker, atenet-router defaults to envoy)", + cfg: routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, }, { name: "atenet-router set to envoy is valid", - cfg: routerConfig{AtenetRouter: string(atenetRouterEnvoy), ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, + cfg: routerConfig{AtenetRouter: string(atenetRouterEnvoy), ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, }, { name: "atenet-router set to agentgateway is valid", - cfg: routerConfig{AtenetRouter: string(atenetRouterAgentgateway), ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, + cfg: routerConfig{AtenetRouter: string(atenetRouterAgentgateway), ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, }, { name: "unknown router rejected", @@ -45,47 +47,64 @@ func TestRouterConfigValidate(t *testing.T) { }, { name: "negative extproc-max-requests rejected", - cfg: routerConfig{ExtProcMaxRequests: -1, ParkedRequest: ParkedRequestConfig{Max: 0}}, + cfg: routerConfig{ExtProcMaxRequests: -1, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, wantErr: "must not be negative", }, { name: "explicit breaker below the lot rejected", - cfg: routerConfig{ExtProcMaxRequests: 512, ParkedRequest: ParkedRequestConfig{Max: 1024}}, + cfg: routerConfig{ExtProcMaxRequests: 512, ParkedRequest: ingress.ParkedRequestConfig{Max: 1024}}, wantErr: "must be >= --parked-request-max", }, { name: "explicit breaker equal to the lot accepted", - cfg: routerConfig{ExtProcMaxRequests: 1024, ParkedRequest: ParkedRequestConfig{Max: 1024}}, + cfg: routerConfig{ExtProcMaxRequests: 1024, ParkedRequest: ingress.ParkedRequestConfig{Max: 1024}}, }, { name: "parking disabled ignores the relation", - cfg: routerConfig{ExtProcMaxRequests: 8, ParkedRequest: ParkedRequestConfig{Max: 0}}, + cfg: routerConfig{ExtProcMaxRequests: 8, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, + }, + { + name: "explicit ingress mode accepted", + cfg: routerConfig{Mode: ModeIngress}, + }, + { + name: "explicit egress mode accepted", + cfg: routerConfig{Mode: ModeEgress}, + }, + { + name: "explicit all mode accepted", + cfg: routerConfig{Mode: ModeAll}, + }, + { + name: "unknown mode rejected", + cfg: routerConfig{Mode: "both"}, + wantErr: `--mode must be one of`, }, { name: "drain-timeout below the parking budget rejected", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 2 * time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 2 * time.Second}, wantErr: "must be >= --parked-request-budget", }, { name: "drain-timeout equal to the parking budget accepted", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 5 * time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 5 * time.Second}, }, { name: "drain-timeout above the parking budget accepted", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 30 * time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 30 * time.Second}, }, { name: "short drain-timeout with parking disabled accepted", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Max: 0}, DrainTimeout: time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Max: 0}, DrainTimeout: time.Second}, }, { name: "negative drain-timeout rejected", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}, DrainTimeout: -time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, DrainTimeout: -time.Second}, wantErr: "--drain-timeout must not be negative", }, { name: "negative drain-delay rejected", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}, DrainDelay: -time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, DrainDelay: -time.Second}, wantErr: "--drain-delay must not be negative", }, } @@ -124,17 +143,42 @@ func TestRouterConfigAtenetRouter(t *testing.T) { } } +// The empty mode is what a routerConfig built in code (rather than from flags) +// carries, and it must behave as ModeAll so nothing silently stops serving. +func TestModeServes(t *testing.T) { + tests := []struct { + mode Mode + wantIngress bool + wantEgress bool + }{ + {mode: "", wantIngress: true, wantEgress: true}, + {mode: ModeAll, wantIngress: true, wantEgress: true}, + {mode: ModeIngress, wantIngress: true, wantEgress: false}, + {mode: ModeEgress, wantIngress: false, wantEgress: true}, + } + for _, tc := range tests { + t.Run(string(tc.mode), func(t *testing.T) { + if got := tc.mode.ServesIngress(); got != tc.wantIngress { + t.Errorf("ServesIngress() = %v, want %v", got, tc.wantIngress) + } + if got := tc.mode.ServesEgress(); got != tc.wantEgress { + t.Errorf("ServesEgress() = %v, want %v", got, tc.wantEgress) + } + }) + } +} + func TestRouterConfigExtProcMaxRequests(t *testing.T) { tests := []struct { name string cfg routerConfig want int }{ - {"auto derives twice the default lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, 2 * defaultParkedRequestMax}, - {"auto scales with a larger lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: 4096}}, 8192}, - {"auto floors at Envoy's default when the lot is small", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: 10}}, extProcMaxRequestsFloor}, - {"auto floors when parking is disabled", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: 0}}, extProcMaxRequestsFloor}, - {"explicit value wins over derivation", routerConfig{ExtProcMaxRequests: 1500, ParkedRequest: ParkedRequestConfig{Max: 1024}}, 1500}, + {"auto derives twice the default lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, 2 * ingress.DefaultParkedRequestMax}, + {"auto scales with a larger lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: 4096}}, 8192}, + {"auto floors at Envoy's default when the lot is small", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: 10}}, extProcMaxRequestsFloor}, + {"auto floors when parking is disabled", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, extProcMaxRequestsFloor}, + {"explicit value wins over derivation", routerConfig{ExtProcMaxRequests: 1500, ParkedRequest: ingress.ParkedRequestConfig{Max: 1024}}, 1500}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -149,19 +193,19 @@ func TestRouterConfigDrainTimeout(t *testing.T) { tests := []struct { name string cfg routerConfig - parkCfg ParkedRequestConfig + parkCfg ingress.ParkedRequestConfig want time.Duration }{ { name: "auto derives budget + route timeout + margin", cfg: routerConfig{DrainTimeout: 0}, - parkCfg: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.normalized(), + parkCfg: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.Normalized(), want: 5*time.Second + defaultRouteTimeout + drainTimeoutMargin, }, { name: "auto scales with a larger budget", cfg: routerConfig{DrainTimeout: 0}, - parkCfg: ParkedRequestConfig{Budget: 30 * time.Second, Max: 1024}.normalized(), + parkCfg: ingress.ParkedRequestConfig{Budget: 30 * time.Second, Max: 1024}.Normalized(), want: 30*time.Second + defaultRouteTimeout + drainTimeoutMargin, }, { @@ -170,13 +214,13 @@ func TestRouterConfigDrainTimeout(t *testing.T) { // normalized() fills Budget even when Max disables parking, so the // derived drain still covers a later re-enable without a restart // surprise. - parkCfg: ParkedRequestConfig{Max: 0}.normalized(), - want: defaultParkedRequestBudget + defaultRouteTimeout + drainTimeoutMargin, + parkCfg: ingress.ParkedRequestConfig{Max: 0}.Normalized(), + want: ingress.DefaultParkedRequestBudget + defaultRouteTimeout + drainTimeoutMargin, }, { name: "explicit value wins over derivation", cfg: routerConfig{DrainTimeout: 42 * time.Second}, - parkCfg: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.normalized(), + parkCfg: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.Normalized(), want: 42 * time.Second, }, } @@ -193,7 +237,7 @@ func TestSetOtlpCollector(t *testing.T) { // No collector address may keep the router from starting. The address // defaults to OTEL_EXPORTER_OTLP_ENDPOINT, which also feeds the router's // own exporter and where https is perfectly valid; the router is the xDS - // control plane for every Envoy in the mesh, so dropping Envoy's spans is + // control plane for every ingress Envoy, so dropping Envoy's spans is // always the cheaper failure. setOtlpCollector returns nothing precisely so // this cannot regress into a startup error. tests := []struct { diff --git a/cmd/atenet/internal/router/controller.go b/cmd/atenet/internal/router/controller.go index e66243d5ba..1554e1051d 100644 --- a/cmd/atenet/internal/router/controller.go +++ b/cmd/atenet/internal/router/controller.go @@ -23,14 +23,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// Controller monitors ActorTemplates and coordinates configuration updates -// for the Envoy xDS and external processing servers. +// Controller monitors ActorTemplates and coordinates configuration updates for +// the ingress Envoy's xDS server. It is part of the ingress control plane and +// only runs in a mode that serves ingress — the egress Envoy is statically +// configured and has no templates to watch. type Controller struct { - k8sClient client.Client - clientset kubernetes.Interface - cfg routerConfig - xdsSrv *XdsServer - extprocSrv *ExtProcServer + k8sClient client.Client + clientset kubernetes.Interface + cfg routerConfig + xdsSrv *XdsServer atStore atStore envoyRunner *envoyrunner @@ -41,7 +42,6 @@ func NewController( clientset kubernetes.Interface, cfg routerConfig, xdsSrv *XdsServer, - extprocSrv *ExtProcServer, ) *Controller { xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr) @@ -53,11 +53,10 @@ func NewController( } return &Controller{ - k8sClient: k8sClient, - clientset: clientset, - cfg: cfg, - xdsSrv: xdsSrv, - extprocSrv: extprocSrv, + k8sClient: k8sClient, + clientset: clientset, + cfg: cfg, + xdsSrv: xdsSrv, atStore: store, envoyRunner: newEnvoyRunner(k8sClient, cfg), diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index af305e5e0f..93bdc5ed5a 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -22,6 +22,8 @@ import ( "time" "golang.org/x/sync/errgroup" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) type dataplaneHealthCheck struct { @@ -44,7 +46,7 @@ func (r atenetRouter) healthCheck() dataplaneHealthCheck { } } -func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) error { +func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ingress.ParkedRequestConfig, traceRootSamplingPercent float64) error { switch s.cfg.atenetRouter() { case atenetRouterEnvoy: s.startEnvoyDataplane(ctx, g, parkCfg, traceRootSamplingPercent) @@ -56,7 +58,7 @@ func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, pa return nil } -func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) { +func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ingress.ParkedRequestConfig, traceRootSamplingPercent float64) { xdsSrv := NewXdsServer(s.cfg.XdsPort) xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr) setOtlpCollector(ctx, xdsSrv, s.cfg.OtlpCollectorAddress) @@ -64,7 +66,7 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou xdsSrv.SetRouteTimeout(s.cfg.RouteTimeout) xdsSrv.SetExtProcMaxRequests(s.cfg.extProcMaxRequests()) - if parkCfg.enabled() { + if parkCfg.Enabled() { // Envoy must keep a parked request open at least as long as the router // will hold it; add a margin so the router surfaces its own 503 first. xdsSrv.SetExtProcMessageTimeout(parkCfg.Budget + 5*time.Second) @@ -72,7 +74,7 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath) xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath, s.cfg.UpstreamSpiffePrefix) - ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv, s.extprocSrv) + ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv) // Envoy receives all routing configuration from the local xDS server. g.Go(func() error { diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go new file mode 100644 index 0000000000..0897e54554 --- /dev/null +++ b/cmd/atenet/internal/router/egress/egress.go @@ -0,0 +1,424 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package egress implements the ext_proc handler for outbound actor traffic: +// it authenticates the actor behind an egress CONNECT before the gateway +// tunnels it out. +// +// The identity this handler acts on comes from the actor certificate presented +// in the mTLS handshake and signed by the actor-identity CA — never from a +// request header. That is the opposite of the ingress package's model, where +// every header is unauthenticated client input. Keeping the two in separate +// packages keeps that difference explicit; the ext_proc mux is what guarantees +// a request only ever reaches the handler for the filter chain that accepted +// it. +package egress + +import ( + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "log/slog" + "net/url" + "slices" + "strings" + "time" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +const ( + // forwardedClientCertHeader is the header Envoy fills in with details of + // the mTLS peer, including the PEM chain it validated. The egress filter + // chain sets forward_client_cert_details: SANITIZE_SET, so whatever a + // client sends under this name is discarded and replaced by Envoy's own + // value. + // + // This is the only channel that can carry a whole certificate to ext_proc: + // the CEL request attributes Envoy exposes (subject, SANs, SHA-256 digest) + // cannot express the custom ActorIdentity X.509 extension this gateway + // authorizes on. + forwardedClientCertHeader = "x-forwarded-client-cert" + // xfccChainKey is the x-forwarded-client-cert key holding the URL-encoded + // PEM of the full presented chain, leaf first. + xfccChainKey = "chain" +) + +// Handler authenticates the actor behind each egress CONNECT. +type Handler struct { + apiClient ateapipb.ControlClient + // actorIdentityRoots is the actor-identity CA bundle every actor + // certificate must chain to. Nil means the gateway cannot authenticate + // anyone, and every CONNECT fails closed. + actorIdentityRoots *x509.CertPool +} + +// New builds the egress handler. actorIdentityRoots is the same trust bundle +// the egress listener uses as its trusted_ca; see verifyActorCertificate for +// why the check is made again here. +func New(apiClient ateapipb.ControlClient, actorIdentityRoots *x509.CertPool) *Handler { + return &Handler{apiClient: apiClient, actorIdentityRoots: actorIdentityRoots} +} + +func (h *Handler) Direction() extproc.Direction { return extproc.DirectionEgress } + +// HandleRequestHeaders authenticates the actor behind an egress CONNECT before +// the gateway tunnels it out, using the actor certificate atunnel presented in +// the mTLS handshake. Nothing the actor can write — no CONNECT header, no +// request metadata — contributes to the identity; the only inputs are the +// certificate the actor-identity CA signed and the control plane's own view of +// that actor. +func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestMetadata) (extproc.Result, error) { + // Sanity check that we were called on the Egress listener filter chain with + // a CONNECT. + if !strings.EqualFold(md.Method, "CONNECT") { + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_MethodNotAllowed, + "egress denied: expected CONNECT, got %q", md.Method) + } + + // No roots means the gateway cannot authenticate anyone. Fail closed, and + // as 503 rather than 403: this is our misconfiguration, not the actor's. + if h.actorIdentityRoots == nil { + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_ServiceUnavailable, + "egress unavailable: no actor-identity CA configured") + } + + identity, err := h.authenticateActorCertificate(md) + if err != nil { + // The body stays generic on purpose: an actor that fails authentication + // has not proven it is anyone, so it gets no detail about why. The + // specific reason rides along as the wrapped cause, which only the + // server-side log below reads. + slog.WarnContext(ctx, "egress denied: actor certificate rejected", slog.Any("err", err)) + return extproc.Result{}, extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, + "egress denied: invalid actor certificate") + } + + if err := validateIdentity(identity); err != nil { + return extproc.Result{}, err + } + if err := h.validateActor(ctx, identity); err != nil { + return extproc.Result{}, err + } + + slog.InfoContext(ctx, "egress identity authenticated", + slog.String("atespace", identity.Atespace), + slog.String("actor", identity.ActorName), + slog.String("actorUid", identity.ActorUid), + // For a CONNECT the :authority is the actor's original destination + // (IP:port). + slog.String("destination", md.Host)) + + // Identity is authenticated; let the CONNECT proceed unchanged. + return extproc.Result{ + Response: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, nil +} + +// validateIdentity checks that the identity a verified actor certificate +// carries names an actor that could exist at all, before it is used as a +// control-plane lookup key. +func validateIdentity(identity *substratex509.ActorIdentity) error { + // The CA only ever mints these from control-plane state, so a name that is + // not a legal resource name means the CA or its inputs are compromised. + if !resources.IsValidResourceName(identity.Atespace) || !resources.IsValidResourceName(identity.ActorName) { + return extproc.NewReqError(envoy_type.StatusCode_Forbidden, + "egress denied: invalid actor identity %q/%q", identity.Atespace, identity.ActorName) + } + return nil +} + +// validateActor checks the identity a certificate certifies against the control +// plane's current view of that actor: it still exists, it is the actor the +// certificate was issued to, and it is running. Every error it returns is +// already a client-facing ext_proc denial. +func (h *Handler) validateActor(ctx context.Context, identity *substratex509.ActorIdentity) error { + atespace := identity.Atespace + actorName := identity.ActorName + actorUID := identity.ActorUid + + // Confirm the certified actor still exists. The name is only a lookup key + // here; the UID below is what actually authorizes. + // TODO: this can cause heavy load on ate api server. Change it based on https://github.com/agent-substrate/substrate/issues/592. + actor, err := h.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: actorName}, + }) + if err != nil { + return mapEgressIdentityError(atespace, actorName, err) + } + + // Authorize on the UID, not the name. The UID the CA certified has to match the UID the + // control plane holds right now. + if uid := actor.GetMetadata().GetUid(); uid != actorUID { + slog.WarnContext(ctx, "egress denied: actor UID mismatch", + slog.String("atespace", atespace), + slog.String("actor", actorName), + slog.String("certificateActorUid", actorUID), + slog.String("currentActorUid", uid)) + return extproc.NewReqError(envoy_type.StatusCode_Forbidden, + "egress denied: actor %q/%q is not the actor this certificate was issued to", atespace, actorName) + } + + // The actor performing egress must actually be running. + if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING { + return extproc.NewReqError(envoy_type.StatusCode_Forbidden, + "egress denied: actor %q/%q is %s, not running", atespace, actorName, actor.GetStatus()) + } + return nil +} + +// authenticateActorCertificate turns the mTLS peer certificate Envoy recorded +// on the request into a verified ActorIdentity, or an error describing why it +// cannot be trusted. +func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*substratex509.ActorIdentity, error) { + header := md.Header(forwardedClientCertHeader) + if header == "" { + return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader) + } + chain, err := parseXFCCChain(header) + if err != nil { + return nil, err + } + return h.verifyActorCertificate(chain) +} + +// verifyActorCertificate checks that chain[0] is a live, non-CA, client-auth +// actor certificate issued by the actor-identity CA, and returns the single +// ActorIdentity it carries. +// +// The chain is verified here even though Envoy already did it at the handshake +// (require_client_certificate with the actor-identity CA as trusted_ca). We have +// to parse the certificate anyway to read the ActorIdentity extension, which +// Envoy cannot see, and trusting a parsed-but-unverified certificate is a +// well-worn source of CVEs. It also keeps the handler safe if the Envoy config +// is ever loosened, and costs one signature check per CONNECT rather than per +// request. The IsCA, ClientAuth-EKU, and purpose checks below have no Envoy-side +// equivalent at all. +func (h *Handler) verifyActorCertificate(chain []*x509.Certificate) (*substratex509.ActorIdentity, error) { + leaf := chain[0] + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + intermediates.AddCert(cert) + } + + now := time.Now() + if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) { + return nil, fmt.Errorf("actor certificate is outside its validity period (%s..%s)", + leaf.NotBefore.Format(time.RFC3339), leaf.NotAfter.Format(time.RFC3339)) + } + // An actor certificate is an end-entity credential. Refusing IsCA here stops + // a leaked or mis-issued CA certificate from being replayed as a leaf: chain + // verification alone would happily accept one. + if leaf.IsCA { + return nil, fmt.Errorf("actor certificate is a CA certificate") + } + // Require ClientAuth explicitly rather than relying on VerifyOptions.KeyUsages: + // an empty ExtKeyUsage means "any usage" to crypto/x509 and would pass. This + // mirrors the check atunnel makes on the certificate when it mints it. + if !slices.Contains(leaf.ExtKeyUsage, x509.ExtKeyUsageClientAuth) { + return nil, fmt.Errorf("actor certificate cannot authenticate a TLS client") + } + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: h.actorIdentityRoots, + Intermediates: intermediates, + CurrentTime: now, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + return nil, fmt.Errorf("actor certificate is not signed by the actor-identity CA: %w", err) + } + + // ActorIdentityFromCertificate returns (nil, nil) when the extension is + // absent, an error when there is more than one or when its contents are + // malformed, empty, or carry a purpose other than atunnel. + identity, err := substratex509.ActorIdentityFromCertificate(leaf) + if err != nil { + return nil, fmt.Errorf("actor certificate has no single valid ActorIdentity extension: %w", err) + } + if identity == nil { + return nil, fmt.Errorf("actor certificate has no ActorIdentity extension") + } + // Restate what ActorIdentityFromCertificate enforces. The gateway is the + // component that gets hurt if that helper ever loosens, and "reject anything + // not scoped to atunnel" is the property this endpoint depends on: a + // certificate minted for some future purpose must not open a tunnel. + if identity.Atespace == "" || identity.ActorName == "" || identity.ActorUid == "" { + return nil, fmt.Errorf("actor certificate identity is incomplete") + } + if identity.Purpose != substratex509.ActorIdentityPurposeAtunnel { + return nil, fmt.Errorf("actor certificate purpose %q is not %q", + identity.Purpose, substratex509.ActorIdentityPurposeAtunnel) + } + return identity, nil +} + +// parseXFCCChain extracts the presented certificate chain, leaf first, from an +// x-forwarded-client-cert header value. +func parseXFCCChain(header string) ([]*x509.Certificate, error) { + // One element per proxy hop. SANITIZE_SET makes Envoy the only writer, so + // anything but exactly one element means either an unexpected proxy in front + // of the gateway or a listener that lost SANITIZE_SET — in both cases we no + // longer know which element describes our actual peer, so refuse to guess. + elements := splitXFCCUnquoted(header, ',') + if len(elements) != 1 { + return nil, fmt.Errorf("expected exactly one %s element, got %d", forwardedClientCertHeader, len(elements)) + } + encoded, ok := xfccValue(elements[0], xfccChainKey) + if !ok { + return nil, fmt.Errorf("%s carries no %q value", forwardedClientCertHeader, xfccChainKey) + } + // Envoy percent-encodes the PEM. PathUnescape, not QueryUnescape: base64 + // bodies contain '+', and query unescaping would decode it to a space and + // silently corrupt the DER. + chainPEM, err := url.PathUnescape(encoded) + if err != nil { + return nil, fmt.Errorf("decoding the client certificate chain: %w", err) + } + + var chain []*x509.Certificate + rest := []byte(chainPEM) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parsing the client certificate chain: %w", err) + } + chain = append(chain, cert) + } + if len(chain) == 0 { + return nil, fmt.Errorf("%s carries no certificate", forwardedClientCertHeader) + } + return chain, nil +} + +// xfccValue returns the value of key in one x-forwarded-client-cert element. +// Keys are matched case-insensitively; Envoy emits "Chain", but the header is +// consumed by enough different proxies that assuming its casing is not worth +// the failure mode. +func xfccValue(element, key string) (string, bool) { + for _, pair := range splitXFCCUnquoted(element, ';') { + k, v, found := strings.Cut(pair, "=") + if !found || !strings.EqualFold(strings.TrimSpace(k), key) { + continue + } + return unquoteXFCC(strings.TrimSpace(v)), true + } + return "", false +} + +// splitXFCCUnquoted splits on sep, ignoring separators inside a quoted value. +// x-forwarded-client-cert quotes any value containing its own delimiters, which +// the PEM ones always do. +func splitXFCCUnquoted(s string, sep rune) []string { + var parts []string + var current strings.Builder + quoted := false + escaped := false + for _, r := range s { + switch { + case escaped: + current.WriteRune(r) + escaped = false + case quoted && r == '\\': + current.WriteRune(r) + escaped = true + case r == '"': + quoted = !quoted + current.WriteRune(r) + case r == sep && !quoted: + parts = append(parts, current.String()) + current.Reset() + default: + current.WriteRune(r) + } + } + parts = append(parts, current.String()) + + trimmed := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + trimmed = append(trimmed, part) + } + } + return trimmed +} + +// unquoteXFCC strips the surrounding quotes from an x-forwarded-client-cert +// value and undoes the backslash escaping inside them. +func unquoteXFCC(value string) string { + if len(value) < 2 || !strings.HasPrefix(value, `"`) || !strings.HasSuffix(value, `"`) { + return value + } + inner := value[1 : len(value)-1] + var out strings.Builder + escaped := false + for _, r := range inner { + if escaped { + out.WriteRune(r) + escaped = false + continue + } + if r == '\\' { + escaped = true + continue + } + out.WriteRune(r) + } + return out.String() +} + +// mapEgressIdentityError converts a GetActor failure into a client-facing +// ext_proc denial. An unknown actor is treated as forbidden (the actor was +// deleted out from under a still-valid certificate); transient control-plane +// failures fail closed with 503. +func mapEgressIdentityError(atespace, actorName string, err error) error { + switch status.Code(err) { + case codes.NotFound: + return extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, + "egress denied: unknown actor %q/%q", atespace, actorName) + case codes.Unavailable, codes.DeadlineExceeded: + return extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, + "egress identity check unavailable for %q/%q: %v", atespace, actorName, err) + default: + return extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, + "egress denied for %q/%q: %v", atespace, actorName, err) + } +} + +// LoadActorIdentityRoots reads the actor-identity CA trust bundle the egress +// gateway verifies actor client certificates against. +func LoadActorIdentityRoots(pemBytes []byte) (*x509.CertPool, error) { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(pemBytes) { + return nil, fmt.Errorf("actor-identity CA bundle contains no certificates") + } + return roots, nil +} diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go new file mode 100644 index 0000000000..1939cd99ac --- /dev/null +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -0,0 +1,594 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/url" + "strings" + "testing" + "time" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +const ( + testEgressAtespace = "default" + testEgressActor = "my-actor" + testEgressActorUID = "1b4e28ba-2fa1-11d2-883f-0016d3cca427" +) + +// testCA is a throwaway CA standing in for the actor-identity CA. +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T, commonName string) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating CA key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: commonName}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing CA certificate: %v", err) + } + return &testCA{cert: cert, key: key} +} + +func (ca *testCA) roots() *x509.CertPool { + pool := x509.NewCertPool() + pool.AddCert(ca.cert) + return pool +} + +// oidActorIdentity mirrors the unexported OID substratex509 encodes the +// ActorIdentity extension under: the Substrate PEN arc, sub-identifier 2. +var oidActorIdentity = append(append(asn1.ObjectIdentifier{}, substratex509.GoogleSubstratePEN...), 2) + +// addActorIdentityUnchecked encodes identity into template the way +// substratex509.AddActorIdentityToCertificate does, minus its validation, so +// tests can mint the malformed identities a real CA would refuse to produce and +// confirm the gateway rejects them anyway. +func addActorIdentityUnchecked(identity *substratex509.ActorIdentity, template *x509.Certificate) error { + value, err := json.Marshal(identity) + if err != nil { + return err + } + template.ExtraExtensions = append(template.ExtraExtensions, pkix.Extension{Id: oidActorIdentity, Value: value}) + return nil +} + +// actorCertOptions mutates the leaf template so each test can break exactly one +// property of an otherwise-valid actor certificate. +type actorCertOptions struct { + identity *substratex509.ActorIdentity + extraIdentity *substratex509.ActorIdentity + mutate func(*x509.Certificate) +} + +// issueActorCert mints a leaf off ca, mirroring what ateapi's actoridentity +// service produces. +func (ca *testCA) issueActorCert(t *testing.T, opts actorCertOptions) *x509.Certificate { + t.Helper() + cert, err := x509.ParseCertificate(ca.issueActorCertDER(t, opts)) + if err != nil { + t.Fatalf("parsing leaf certificate: %v", err) + } + return cert +} + +// issueActorCertDER is issueActorCert without the parse, for the certificates +// crypto/x509 itself refuses to read back. +func (ca *testCA) issueActorCertDER(t *testing.T, opts actorCertOptions) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: testEgressActor}, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: false, + } + identity := opts.identity + if identity == nil { + identity = &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + } + } + // AddActorIdentityToCertificate validates its input, so identities a real CA + // would refuse to mint are encoded directly. + if err := addActorIdentityUnchecked(identity, template); err != nil { + t.Fatalf("adding ActorIdentity extension: %v", err) + } + if opts.extraIdentity != nil { + if err := addActorIdentityUnchecked(opts.extraIdentity, template); err != nil { + t.Fatalf("adding second ActorIdentity extension: %v", err) + } + } + if opts.mutate != nil { + opts.mutate(template) + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("signing leaf certificate: %v", err) + } + return der +} + +// xfccHeader renders chain the way Envoy's SANITIZE_SET + +// set_current_client_cert_details{chain: true} does. +func xfccHeader(chain ...*x509.Certificate) string { + der := make([][]byte, 0, len(chain)) + for _, cert := range chain { + der = append(der, cert.Raw) + } + return xfccHeaderDER(der...) +} + +func xfccHeaderDER(chain ...[]byte) string { + var buf strings.Builder + for _, der := range chain { + _ = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}) + } + return fmt.Sprintf(`By=spiffe://cluster.local/ns/ate-system/sa/atenet-egress;Hash=abc123;Chain="%s"`, + url.PathEscape(buf.String())) +} + +// egressHandler builds a Handler whose GetActor returns actor/err. +func egressHandler(roots *x509.CertPool, actor *ateapipb.Actor, err error) *Handler { + return New(&egressMockClient{actor: actor, err: err}, roots) +} + +type egressMockClient struct { + ateapipb.ControlClient + actor *ateapipb.Actor + err error +} + +func (m *egressMockClient) GetActor(context.Context, *ateapipb.GetActorRequest, ...grpc.CallOption) (*ateapipb.Actor, error) { + if m.err != nil { + return nil, m.err + } + return m.actor, nil +} + +func runningActor() *ateapipb.Actor { + return &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testEgressAtespace, + Name: testEgressActor, + Uid: testEgressActorUID, + }, + Status: ateapipb.Actor_STATUS_RUNNING, + } +} + +// egressMetadata builds the CONNECT the egress listener hands to ext_proc. +func egressMetadata(xfcc string) *extproc.RequestMetadata { + headers := []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("CONNECT")}, + {Key: ":authority", RawValue: []byte("93.184.216.34:80")}, + } + if xfcc != "" { + headers = append(headers, &corev3.HeaderValue{Key: forwardedClientCertHeader, RawValue: []byte(xfcc)}) + } + return extproc.NewRequestMetadata(headers) +} + +func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) { + t.Helper() + if err == nil { + t.Fatalf("expected a denial with status %d, got none", want) + } + var re *extproc.ReqError + if !errors.As(err, &re) { + t.Fatalf("error %v is not a *extproc.ReqError", err) + } + if re.StatusCode != int(want) { + t.Errorf("status = %d, want %d (%v)", re.StatusCode, want, err) + } +} + +func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + h := egressHandler(ca.roots(), runningActor(), nil) + + res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v, want nil", err) + } + if res.Response == nil { + t.Fatal("HandleRequestHeaders() returned no response") + } + // Egress neither resumes an actor nor picks an upstream. + if res.Resume != "" { + t.Errorf("resume outcome = %q, want %q", res.Resume, "") + } + if res.Target != "" { + t.Errorf("target = %q, want %q", res.Target, "") + } +} + +// Every way an actor certificate can fail to prove an identity has to end in a +// denial, never in a tunnel. +func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + otherCA := newTestCA(t, "some-other-ca") + + tests := []struct { + name string + xfcc func(t *testing.T) string + want envoy_type.StatusCode + }{ + { + name: "no client certificate at all", + xfcc: func(*testing.T) string { return "" }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "signed by an unknown CA", + xfcc: func(t *testing.T) string { + return xfccHeader(otherCA.issueActorCert(t, actorCertOptions{})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "expired", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.NotBefore = time.Now().Add(-2 * time.Hour) + c.NotAfter = time.Now().Add(-time.Hour) + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "not yet valid", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.NotBefore = time.Now().Add(time.Hour) + c.NotAfter = time.Now().Add(2 * time.Hour) + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "no ClientAuth EKU", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + // An empty EKU means "any usage" to crypto/x509 and would pass + // VerifyOptions.KeyUsages; the explicit ClientAuth check is what + // catches it. + name: "empty EKU", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.ExtKeyUsage = nil + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "is a CA certificate", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.IsCA = true + c.KeyUsage |= x509.KeyUsageCertSign + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "no ActorIdentity extension", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.ExtraExtensions = nil + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + // Stays in DER: crypto/x509 refuses to parse a certificate with a + // duplicate extension OID at all, so the helper cannot hand back an + // *x509.Certificate here. That refusal is the first of the two + // layers guarding "exactly one ActorIdentity" — this case proves the + // handler denies rather than panics when the parse fails, and + // substratex509 rejects a second copy if a parser ever allowed one. + name: "two ActorIdentity extensions", + xfcc: func(t *testing.T) string { + return xfccHeaderDER(ca.issueActorCertDER(t, actorCertOptions{ + extraIdentity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: "a-different-actor", + ActorUid: "8f14e45f-ceea-467a-9575-25a0d5d5e4b0", + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }, + })) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "generic purpose", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + Purpose: "generic", + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "missing purpose", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "empty atespace", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "empty actor name", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorUid: testEgressActorUID, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "empty actor UID", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + // SANITIZE_SET makes Envoy the only writer of the header, so two + // elements means we can no longer tell which one is our peer. + name: "two XFCC elements", + xfcc: func(t *testing.T) string { + leaf := ca.issueActorCert(t, actorCertOptions{}) + return xfccHeader(leaf) + "," + xfccHeader(leaf) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "XFCC without a Chain value", + xfcc: func(*testing.T) string { + return `By=spiffe://cluster.local/ns/ate-system/sa/atenet-egress;Hash=abc123` + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "XFCC Chain that is not a certificate", + xfcc: func(*testing.T) string { + return `Chain="` + url.PathEscape("-----BEGIN CERTIFICATE-----\nbm90LWEtY2VydA==\n-----END CERTIFICATE-----\n") + `"` + }, + want: envoy_type.StatusCode_Forbidden, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := egressHandler(ca.roots(), runningActor(), nil) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(tc.xfcc(t))) + wantStatus(t, err, tc.want) + }) + } +} + +// The certificate authenticates; these cover what the control plane says about +// the actor it names. +func TestHandleRequestHeadersAuthorization(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + + tests := []struct { + name string + actor *ateapipb.Actor + err error + want envoy_type.StatusCode + }{ + { + // The actor was deleted and recreated under the same name: the + // certificate is still cryptographically valid but names a UID that + // no longer exists, and must not carry over to the successor. + name: "actor UID does not match the certificate", + actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testEgressAtespace, + Name: testEgressActor, + Uid: "d41d8cd9-8f00-4204-a980-0998ecf8427e", + }, + Status: ateapipb.Actor_STATUS_RUNNING, + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "actor has no UID", + actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testEgressAtespace, Name: testEgressActor}, + Status: ateapipb.Actor_STATUS_RUNNING, + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "actor is not running", + actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testEgressAtespace, + Name: testEgressActor, + Uid: testEgressActorUID, + }, + Status: ateapipb.Actor_STATUS_SUSPENDED, + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "actor no longer exists", + err: status.Error(codes.NotFound, "no such actor"), + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "control plane unreachable", + err: status.Error(codes.Unavailable, "ateapi is down"), + want: envoy_type.StatusCode_ServiceUnavailable, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := egressHandler(ca.roots(), tc.actor, tc.err) + leaf := ca.issueActorCert(t, actorCertOptions{}) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + wantStatus(t, err, tc.want) + }) + } +} + +// An ingress-only router has no actor-identity CA. If an egress CONNECT somehow +// reaches it, it must fail closed rather than tunnel unauthenticated traffic. +func TestHandleRequestHeadersWithoutConfiguredCA(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + h := egressHandler(nil, runningActor(), nil) + + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) +} + +func TestHandleRequestHeadersRejectsNonConnect(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + h := egressHandler(ca.roots(), runningActor(), nil) + + md := egressMetadata(xfccHeader(leaf)) + md.Method = "GET" + md.Headers[":method"] = "GET" + + _, err := h.HandleRequestHeaders(context.Background(), md) + wantStatus(t, err, envoy_type.StatusCode_MethodNotAllowed) +} + +// PEM bodies routinely contain '+'. Decoding the header as a query string would +// turn those into spaces and corrupt the DER, so pin the round trip. +func TestParseXFCCChainPreservesPlusInPEM(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + // Serials differ per certificate, so mint until one encodes with a '+'. + var leaf *x509.Certificate + for i := 0; i < 50; i++ { + candidate := ca.issueActorCert(t, actorCertOptions{}) + if strings.Contains(string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: candidate.Raw})), "+") { + leaf = candidate + break + } + } + if leaf == nil { + t.Skip("no certificate with a '+' in its PEM body after 50 attempts") + } + + chain, err := parseXFCCChain(xfccHeader(leaf)) + if err != nil { + t.Fatalf("parseXFCCChain() error = %v", err) + } + if len(chain) != 1 || !chain[0].Equal(leaf) { + t.Fatalf("parseXFCCChain() did not round-trip the certificate") + } +} + +func TestParseXFCCChainIncludesIntermediates(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + + chain, err := parseXFCCChain(xfccHeader(leaf, ca.cert)) + if err != nil { + t.Fatalf("parseXFCCChain() error = %v", err) + } + if len(chain) != 2 { + t.Fatalf("parseXFCCChain() returned %d certificates, want 2", len(chain)) + } + if !chain[0].Equal(leaf) { + t.Error("parseXFCCChain() did not return the leaf first") + } +} diff --git a/cmd/atenet/internal/router/extproc.go b/cmd/atenet/internal/router/extproc.go deleted file mode 100644 index a668a813d6..0000000000 --- a/cmd/atenet/internal/router/extproc.go +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "context" - "errors" - "fmt" - "io" - "log/slog" - "net" - "time" - - "github.com/agent-substrate/substrate/internal/ateattr" - extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" - envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/propagation" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" -) - -// ExtProcServer implements the Envoy external processing gRPC server -// to dynamically manage actor activations based on request traffic. -type ExtProcServer struct { - port int - apiClient ateapipb.ControlClient - recorder *QueryRecorder - resumer *ActorResumer - routeDuration metric.Float64Histogram - parking *parkingLot - routeViaAuthority bool -} - -func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool) *ExtProcServer { - return &ExtProcServer{ - port: port, - apiClient: apiClient, - recorder: NewQueryRecorder(100), - resumer: NewActorResumer(apiClient, withParking(parkCfg)), - routeDuration: routeDuration, - parking: newParkingLot(parkCfg, parkMetrics), - routeViaAuthority: routeViaAuthority, - } -} - -// NewGRPCServer builds the gRPC server with the ext_proc service registered. -// The caller owns its lifecycle: Run serves it, and the drain sequence in -// drain.go stops it — gracefully first so in-flight streams (parked requests -// above all) finish, forcefully past the drain timeout. -func (s *ExtProcServer) NewGRPCServer() *grpc.Server { - grpcServer := grpc.NewServer( - grpc.StatsHandler(otelgrpc.NewServerHandler()), - ) - extprocv3.RegisterExternalProcessorServer(grpcServer, s) - return grpcServer -} - -func (s *ExtProcServer) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { - for { - req, err := stream.Recv() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - - resp := &extprocv3.ProcessingResponse{} - - switch reqType := req.Request.(type) { - case *extprocv3.ProcessingRequest_RequestHeaders: - start := time.Now() - hResponse, rqm, target, tmplNs, tmplName, resumeOutcome, err := s.handleRequestHeaders(stream.Context(), reqType.RequestHeaders) - elapsed := time.Since(start) - outcomeStr := classifyOutcome(err) - resumeStr := string(resumeOutcome) - if err != nil { - slog.ErrorContext(stream.Context(), "Error during ext_proc RequestHeaders processing", slog.String("err", err.Error())) - var reqErr *reqError - if errors.As(err, &reqErr) { - resp = immediateResponse(envoy_type.StatusCode(reqErr.statusCode), reqErr.Error()) - } else { - resp = immediateResponse(envoy_type.StatusCode_InternalServerError, err.Error()) - } - s.recordRouteDuration(stream.Context(), elapsed, tmplNs, tmplName, outcomeStr, resumeStr) - s.recorder.AddRouterRequest(start, elapsed, "Error", "-", rqm) - } else { - resp.Response = &extprocv3.ProcessingResponse_RequestHeaders{RequestHeaders: hResponse} - s.recordRouteDuration(stream.Context(), elapsed, tmplNs, tmplName, outcomeStr, resumeStr) - s.recorder.AddRouterRequest(start, elapsed, "Route ok", target, rqm) - } - - default: - // No modification for other processing states, but log because this should - // not be called. - slog.Error("Unexpected request type", slog.String("reqType", fmt.Sprintf("%T", reqType))) - resp.Response = &extprocv3.ProcessingResponse_RequestHeaders{ - RequestHeaders: &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{}, - }, - } - } - - if err := stream.Send(resp); err != nil { - return err - } - } -} - -func (s *ExtProcServer) handleRequestHeaders( - ctx context.Context, - reqHeaders *extprocv3.HttpHeaders, -) (*extprocv3.HeadersResponse, *requestMetadata, string, string, string, ResumeOutcome, error) { - metadata := newRequestMetadata(reqHeaders.Headers.GetHeaders()) - slog.InfoContext(ctx, "Request", slog.String("host", metadata.host)) - - // Envoy doesn't propagate trace context into the ext_proc gRPC - // stream's metadata — the per-request traceparent arrives in the - // HTTP headers carried inside the ProcessingRequest payload. Extract - // from there so our span links to the Envoy ingress span. - ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(metadata.headers)) - ctx, span := otel.Tracer(routerServiceName).Start(ctx, "ExtProc.RequestHeaders") - defer span.End() - - actorRef, err := parseActorRef(metadata.host) - if err != nil { - // Host is invalid, respond with 404. - return nil, metadata, "", "", "", ResumeOutcomeNone, invalidHostErr(metadata.host, err) - } - - // Admit the request to the parking lot before resuming. While resume is - // in-flight the request occupies a slot; if the actor's worker pool is - // momentarily saturated the resumer parks (retries) here rather than failing - // fast. A full lot sheds the request immediately so the router applies - // backpressure instead of queueing without bound. - release, ok := s.parking.enter(ctx) - if !ok { - return nil, metadata, "", "", "", ResumeOutcomeNone, parkingFullErr(actorRef.String()) - } - - slog.InfoContext(ctx, "ResumeActor", slog.Any("actor", actorRef)) - actor, resumeOutcome, err := s.resumer.ResumeActor(ctx, actorRef) - release(parkOutcomeFor(err)) - if err != nil { - return nil, metadata, "", "", "", resumeOutcome, mapResumeError(actorRef, err) - } - - // Actor template identity, used as low-cardinality route-latency metric - // attributes (see recordRouteDuration). - tmplNs := actor.GetActorTemplateNamespace() - tmplName := actor.GetActorTemplateName() - - workerIP := actor.GetWorkerAssignment().GetWorkerPodIp() - slog.InfoContext(ctx, "ResumeActor result", - slog.Any("actor", actorRef), - slog.String("status", actor.GetStatus().String()), - slog.String("workerIP", workerIP)) - - if ip := net.ParseIP(workerIP); ip == nil { - return nil, metadata, "", tmplNs, tmplName, resumeOutcome, newReqError(envoy_type.StatusCode_InternalServerError, - "actor %s routing failed", actorRef) - } - - // The actor is reached through the in-worker atunnel ingress server, which - // listens on :443 (mTLS) and forwards to the actor's :80. The worker no - // longer DNATs pod-IP:80 to the actor, so the router dials :443 and the - // ORIGINAL_DST cluster's upstream TLS context presents the router's - // podidentity client cert (see buildOriginalDstCluster and - // buildUpstreamTransportSocket). - // TODO(bowei) -- handle more than port 80 on the actor. - targetAddr := net.JoinHostPort(workerIP, "443") - - slog.InfoContext(ctx, "Route ok", slog.Any("actor", actorRef), slog.String("targetAddr", targetAddr)) - - // Route by telling the ORIGINAL_DST cluster which worker atunnel address to - // dial, without touching :authority — atunnel authorizes the actor by the - // original Host (actor DNS name). - mutation := &extprocv3.HeaderMutation{} - addRoutingMutations(targetAddr, metadata.host, s.routeViaAuthority, mutation) - - return &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{ - HeaderMutation: mutation, - }, - }, metadata, targetAddr, tmplNs, tmplName, resumeOutcome, nil -} - -func (s *ExtProcServer) recordRouteDuration(ctx context.Context, d time.Duration, tmplNs, tmplName, outcome, resume string) { - if s.routeDuration == nil { - return - } - s.routeDuration.Record(ctx, d.Seconds(), metric.WithAttributes( - ateattr.TemplateNamespaceKey.String(tmplNs), - ateattr.TemplateNameKey.String(tmplName), - ateattr.RouterOutcomeKey.String(outcome), - ateattr.RouterResumeKey.String(resume), - )) -} - -func classifyOutcome(err error) string { - if err == nil { - return "ok" - } - if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { - return "cancelled" - } - if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { - return "timeout" - } - switch status.Code(err) { - case codes.FailedPrecondition: - return "no_capacity" - case codes.Aborted: - return "lock_conflict" - case codes.NotFound: - return "not_found" - case codes.Unavailable: - return "unavailable" - case codes.ResourceExhausted: - return "rate_limited" - } - var re *reqError - if errors.As(err, &re) { - switch envoy_type.StatusCode(re.statusCode) { - case envoy_type.StatusCode_NotFound: - return "not_found" - case envoy_type.StatusCode_ServiceUnavailable: - return "no_capacity" - case envoy_type.StatusCode_GatewayTimeout: - return "timeout" - case envoy_type.StatusCode_TooManyRequests: - return "rate_limited" - } - } - return "resume_error" -} diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go new file mode 100644 index 0000000000..52864d9cd9 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" +) + +// Direction is the gateway a request arrived through. It selects the handler, +// so it must come from something the dataplane asserts rather than from the +// request. +type Direction string + +const ( + // DirectionIngress is inbound traffic addressed to an actor. + DirectionIngress Direction = "ingress" + // DirectionEgress is outbound traffic tunneled out of an actor. + DirectionEgress Direction = "egress" +) + +const ( + // EgressFilterChainName is the Envoy filter chain that terminates actor + // egress CONNECTs, and so the one that selects the egress handler. It must + // stay in sync with the filter chain name in + // manifests/ate-install/atenet-egress.yaml. + EgressFilterChainName = "egress" + // FilterChainNameAttribute is the CEL attribute carrying the name of the + // filter chain that accepted the request. The egress Envoy asks for it via + // request_attributes on its ext_proc filter. + // + // Do not "improve" this to xds.listener_name: Envoy 1.34 cannot parse that + // one, and rather than failing config load it logs "error parsing cel + // expression" at trace level and sends an empty attributes map. An absent + // attribute means ingress here, so every egress CONNECT would silently take + // the ingress path and 404 on the actor DNS name parse. + FilterChainNameAttribute = "xds.filter_chain_name" +) + +// directionOf reports which direction's handler an ext_proc RequestHeaders +// callback belongs to. +// +// Dispatch is by filter chain, not by :method, because the two directions apply +// opposite trust models: on egress the actor identity comes from a client +// certificate Envoy validated against the actor-identity CA, while on ingress +// every request header is unauthenticated client input. Keying on :method would +// let any external client sending CONNECT select the egress handler and use its +// denial messages as an actor-existence and status oracle. Envoy asserts the +// filter chain name; the request cannot influence it. +// +// An unrecognized or absent attribute means ingress, the fail-safe direction: +// an egress request misrouted to the ingress handler fails to parse as an actor +// DNS name and 404s, whereas the reverse leaks control-plane state. +func directionOf(req *extprocv3.ProcessingRequest) Direction { + if filterChainName(req) == EgressFilterChainName { + return DirectionEgress + } + return DirectionIngress +} + +// filterChainName returns the xds.filter_chain_name attribute Envoy attached to +// the request, or "" when the listener did not request the attribute. The +// attributes map is keyed by the ext_proc filter's name within the HCM chain, +// which we do not want to hardcode here, so scan every entry. +func filterChainName(req *extprocv3.ProcessingRequest) string { + for _, attrs := range req.GetAttributes() { + if v, ok := attrs.GetFields()[FilterChainNameAttribute]; ok { + return v.GetStringValue() + } + } + return "" +} diff --git a/cmd/atenet/internal/router/extproc/dispatch_test.go b/cmd/atenet/internal/router/extproc/dispatch_test.go new file mode 100644 index 0000000000..83252eb6d5 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/dispatch_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "google.golang.org/protobuf/types/known/structpb" +) + +// connectRequest builds a RequestHeaders ProcessingRequest for a CONNECT, +// optionally attributed to a filter chain. filterKey is the ext_proc filter +// name Envoy keys the attributes map by. +func connectRequest(filterKey, chain string) *extprocv3.ProcessingRequest { + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{ + Headers: []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("CONNECT")}, + {Key: ":authority", RawValue: []byte("10.0.0.9:443")}, + }, + }, + }, + }, + } + if chain != "" { + req.Attributes = map[string]*structpb.Struct{ + filterKey: { + Fields: map[string]*structpb.Value{ + FilterChainNameAttribute: structpb.NewStringValue(chain), + }, + }, + } + } + return req +} + +// The ingress listener names the router's xDS server assigns. Spelled out here +// rather than imported: the point of this package is that the mux knows nothing +// about either direction's configuration, so the test may not reach for it +// either. +const ( + ingressHTTPListener = "ingress_http_listener" + ingressHTTPSListener = "ingress_https_listener" +) + +func TestDirectionOf(t *testing.T) { + tests := []struct { + name string + filterKey string + chain string + want Direction + }{ + { + name: "egress filter chain", + filterKey: "envoy.filters.http.ext_proc", + chain: EgressFilterChainName, + want: DirectionEgress, + }, + { + name: "egress filter chain under a renamed filter", + filterKey: "some.custom.ext_proc.name", + chain: EgressFilterChainName, + want: DirectionEgress, + }, + { + // The pre-listener-dispatch hole: an external client sending + // CONNECT to the ingress gateway must not reach the egress handler, + // whose denials would otherwise report whether an arbitrary actor + // exists and is running. + name: "CONNECT on the ingress HTTP listener", + filterKey: "envoy.filters.http.ext_proc", + chain: ingressHTTPListener, + want: DirectionIngress, + }, + { + name: "CONNECT on the ingress HTTPS listener", + filterKey: "envoy.filters.http.ext_proc", + chain: ingressHTTPSListener, + want: DirectionIngress, + }, + { + // A listener that never requested the attribute falls back to + // ingress, the fail-safe direction. + name: "no attributes at all", + chain: "", + want: DirectionIngress, + }, + { + name: "unrecognised filter chain", + filterKey: "envoy.filters.http.ext_proc", + chain: "some-other-chain", + want: DirectionIngress, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := directionOf(connectRequest(tc.filterKey, tc.chain)); got != tc.want { + t.Errorf("directionOf() = %v, want %v", got, tc.want) + } + }) + } +} + +// A request the client dresses up to look like egress must not be enough: only +// the Envoy-asserted filter chain name selects the egress handler. +func TestDirectionOfIgnoresClientSuppliedAttributeHeader(t *testing.T) { + req := connectRequest("envoy.filters.http.ext_proc", ingressHTTPListener) + rh := req.GetRequestHeaders().GetHeaders() + rh.Headers = append(rh.Headers, + &corev3.HeaderValue{Key: FilterChainNameAttribute, RawValue: []byte(EgressFilterChainName)}, + &corev3.HeaderValue{Key: "x-envoy-filter-chain-name", RawValue: []byte(EgressFilterChainName)}, + ) + + if got := directionOf(req); got != DirectionIngress { + t.Errorf("directionOf() = %v for a client-forged filter chain header, want %v", got, DirectionIngress) + } +} diff --git a/cmd/atenet/internal/router/extproc/errors.go b/cmd/atenet/internal/router/extproc/errors.go new file mode 100644 index 0000000000..0a6acb19aa --- /dev/null +++ b/cmd/atenet/internal/router/extproc/errors.go @@ -0,0 +1,82 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "fmt" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +// ReqError is a handler's denial of a request: an HTTP-mappable status code +// and a client-safe message the mux turns into an immediate response. The +// underlying cause (if any) is preserved via Unwrap so logs can inspect the +// full chain without leaking server-side detail into the response body. +type ReqError struct { + Msg string + Cause error + StatusCode int +} + +func (e *ReqError) Error() string { return e.Msg } +func (e *ReqError) Unwrap() error { return e.Cause } + +// NewReqError builds a ReqError whose body is the formatted message and no +// wrapped cause. Use WrapReqError when a cause is available. +func NewReqError(code envoy_type.StatusCode, format string, args ...any) error { + return &ReqError{ + Msg: fmt.Sprintf(format, args...), + StatusCode: int(code), + } +} + +// WrapReqError builds a ReqError that keeps cause reachable through Unwrap +// while answering the client with only the formatted message. +func WrapReqError(code envoy_type.StatusCode, cause error, format string, args ...any) error { + return &ReqError{ + Msg: fmt.Sprintf(format, args...), + Cause: cause, + StatusCode: int(code), + } +} + +// ImmediateResponse tells the dataplane to answer the request itself, without +// going upstream. +func ImmediateResponse(statusCode envoy_type.StatusCode, message string) *extprocv3.ProcessingResponse { + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ImmediateResponse{ + ImmediateResponse: &extprocv3.ImmediateResponse{ + Status: &envoy_type.HttpStatus{ + Code: statusCode, + }, + Body: []byte(message), + Headers: &extprocv3.HeaderMutation{ + SetHeaders: []*corev3.HeaderValueOption{ + { + // Using RawValues instead of Value: newer versions of Envoy + // drop Value and use RawValue + Header: &corev3.HeaderValue{ + Key: "content-type", + RawValue: []byte("text/plain"), + }, + }, + }, + }, + }, + }, + } +} diff --git a/cmd/atenet/internal/router/extproc/errors_test.go b/cmd/atenet/internal/router/extproc/errors_test.go new file mode 100644 index 0000000000..ad8da7fe30 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/errors_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "errors" + "testing" + + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +func TestNewReqError(t *testing.T) { + t.Parallel() + + err := NewReqError(envoy_type.StatusCode_BadRequest, "actor %q is %s", "abc", "bad") + if err == nil { + t.Fatal("NewReqError returned nil") + } + var reqErr *ReqError + if !errors.As(err, &reqErr) { + t.Fatalf("errors.As(*ReqError) = false, want true; err type = %T", err) + } + if reqErr.StatusCode != int(envoy_type.StatusCode_BadRequest) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_BadRequest) + } + if got, want := err.Error(), `actor "abc" is bad`; got != want { + t.Errorf("Error() = %q, want %q", got, want) + } +} + +// TestImmediateResponseHeaderEncoding pins the RawValue encoding: Envoy drops +// plain Value in ext_proc header mutations, so a Value-encoded header reaches +// the client with an empty value (found live — content-type on every immediate +// response had been arriving empty). +func TestImmediateResponseHeaderEncoding(t *testing.T) { + t.Parallel() + + resp := ImmediateResponse(envoy_type.StatusCode_ServiceUnavailable, "body") + set := resp.GetImmediateResponse().GetHeaders().GetSetHeaders() + if len(set) != 1 { + t.Fatalf("SetHeaders count = %d, want 1", len(set)) + } + h := set[0].GetHeader() + if h.GetKey() != "content-type" || string(h.GetRawValue()) != "text/plain" { + t.Errorf("header = %q:%q (RawValue), want content-type:text/plain", h.GetKey(), h.GetRawValue()) + } + if h.GetValue() != "" { + t.Errorf("header uses Value (%q); must use RawValue only", h.GetValue()) + } +} diff --git a/cmd/atenet/internal/router/extproc/extproc.go b/cmd/atenet/internal/router/extproc/extproc.go new file mode 100644 index 0000000000..74153f3259 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/extproc.go @@ -0,0 +1,177 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extproc implements the external processing (ext_proc) gRPC server +// that the atenet router serves to its dataplane gateways. +// +// This package is the multiplexer and nothing else: it terminates the +// ext_proc stream, works out which direction — ingress or egress — a request +// arrived on, dispatches to the Handler registered for that direction, and +// records the latency and outcome. The routing and authentication policy for +// each direction lives in the sibling ingress and egress packages, which apply +// opposite trust models and are deliberately kept apart. +package extproc + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "time" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel/metric" + "google.golang.org/grpc" +) + +// Handlers maps each direction this server serves to the handler for it. A +// direction absent from the map is refused at the mux — see Server.Process. +type Handlers map[Direction]Handler + +// Server implements the external processing gRPC server, dispatching each +// request to the Handler for the direction it arrived on. +type Server struct { + port int + handlers Handlers + recorder *QueryRecorder + routeDuration metric.Float64Histogram +} + +// NewServer builds the ext_proc mux serving the given handlers. Passing a +// subset of the directions is how --mode restricts an instance to the traffic +// its deployment fronts. +func NewServer(port int, routeDuration metric.Float64Histogram, handlers Handlers) *Server { + return &Server{ + port: port, + handlers: handlers, + recorder: NewQueryRecorder(100), + routeDuration: routeDuration, + } +} + +// Queries returns the most recently processed requests, newest first, for the +// /statusz page. +func (s *Server) Queries() []RecordedQuery { + return s.recorder.Get() +} + +// Recorder exposes the query ring buffer so tests and the status page can seed +// or read it. +func (s *Server) Recorder() *QueryRecorder { return s.recorder } + +// NewGRPCServer builds the gRPC server with the ext_proc service registered. +// The caller owns its lifecycle: Run serves it, and the drain sequence in +// drain.go stops it — gracefully first so in-flight streams (parked requests +// above all) finish, forcefully past the drain timeout. +func (s *Server) NewGRPCServer() *grpc.Server { + grpcServer := grpc.NewServer( + grpc.StatsHandler(otelgrpc.NewServerHandler()), + ) + extprocv3.RegisterExternalProcessorServer(grpcServer, s) + return grpcServer +} + +func (s *Server) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + var resp *extprocv3.ProcessingResponse + + switch reqType := req.Request.(type) { + case *extprocv3.ProcessingRequest_RequestHeaders: + resp = s.processRequestHeaders(stream.Context(), req, reqType.RequestHeaders) + + default: + // No modification for other processing states, but log because this should + // not be called. + slog.Error("Unexpected request type", slog.String("reqType", fmt.Sprintf("%T", reqType))) + resp = &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, + } + } + + if err := stream.Send(resp); err != nil { + return err + } + } +} + +// processRequestHeaders runs one RequestHeaders callback through the handler +// for its direction, and records the latency and outcome either way. +func (s *Server) processRequestHeaders( + ctx context.Context, + req *extprocv3.ProcessingRequest, + reqHeaders *extprocv3.HttpHeaders, +) *extprocv3.ProcessingResponse { + start := time.Now() + md := NewRequestMetadata(reqHeaders.GetHeaders().GetHeaders()) + + // One atenet binary serves both directions, as two ext_proc handlers + // selected here. They are deployed separately today — atenet-router fronts + // the ingress dataplane, atenet-egress the egress gateway — because the two + // scale independently, and --mode restricts an instance to the direction + // its deployment fronts. Nothing stops a single instance from serving both. + // + // Which handler runs is decided by the filter chain the dataplane says + // accepted the request, never by anything in the request itself (see + // directionOf). + dir := directionOf(req) + + var res Result + var err error + if handler, ok := s.handlers[dir]; ok { + res, err = handler.HandleRequestHeaders(ctx, md) + } else { + // The dataplane in front of this instance is sending traffic the + // instance was not started to serve. Refuse it rather than falling back + // to the other direction's handler: the two apply opposite trust + // models, so a fallback would run a request through the wrong one. + err = NewReqError(envoy_type.StatusCode_NotFound, + "this router does not serve %s traffic", dir) + } + + elapsed := time.Since(start) + s.recordRouteDuration(ctx, elapsed, res.TemplateNamespace, res.TemplateName, classifyOutcome(err), res.resume()) + + if err != nil { + slog.ErrorContext(ctx, "Error during ext_proc RequestHeaders processing", + slog.String("direction", string(dir)), + slog.String("err", err.Error())) + s.recorder.AddRouterRequest(start, elapsed, "Error", "-", md) + + var reqErr *ReqError + if errors.As(err, &reqErr) { + return ImmediateResponse(envoy_type.StatusCode(reqErr.StatusCode), reqErr.Error()) + } + return ImmediateResponse(envoy_type.StatusCode_InternalServerError, err.Error()) + } + + s.recorder.AddRouterRequest(start, elapsed, "Route ok", res.Target, md) + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{RequestHeaders: res.Response}, + } +} diff --git a/cmd/atenet/internal/router/extproc/extproc_test.go b/cmd/atenet/internal/router/extproc/extproc_test.go new file mode 100644 index 0000000000..384b910ccd --- /dev/null +++ b/cmd/atenet/internal/router/extproc/extproc_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + "strings" + "testing" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +// stubHandler records that it ran and returns an empty successful Result. +type stubHandler struct { + direction Direction + called bool +} + +func (h *stubHandler) Direction() Direction { return h.direction } + +func (h *stubHandler) HandleRequestHeaders(context.Context, *RequestMetadata) (Result, error) { + h.called = true + return Result{Response: &extprocv3.HeadersResponse{Response: &extprocv3.CommonResponse{}}}, nil +} + +// The mux must pick the handler by the Envoy-asserted filter chain, and refuse +// outright when this instance was not started to serve that direction (--mode). +// Falling back to the other handler would run the request through the opposite +// trust model. +func TestProcessRequestHeadersDispatchesByMode(t *testing.T) { + tests := []struct { + name string + registered []Direction + chain string + wantRan Direction + wantStatus envoy_type.StatusCode // 0 means "expect success" + }{ + { + name: "both directions served, egress chain", + registered: []Direction{DirectionIngress, DirectionEgress}, + chain: EgressFilterChainName, + wantRan: DirectionEgress, + }, + { + name: "both directions served, ingress chain", + registered: []Direction{DirectionIngress, DirectionEgress}, + chain: ingressHTTPListener, + wantRan: DirectionIngress, + }, + { + name: "egress-only instance refuses ingress traffic", + registered: []Direction{DirectionEgress}, + chain: ingressHTTPListener, + wantStatus: envoy_type.StatusCode_NotFound, + }, + { + name: "ingress-only instance refuses egress traffic", + registered: []Direction{DirectionIngress}, + chain: EgressFilterChainName, + wantStatus: envoy_type.StatusCode_NotFound, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + handlers := Handlers{} + stubs := map[Direction]*stubHandler{} + for _, d := range tc.registered { + stubs[d] = &stubHandler{direction: d} + handlers[d] = stubs[d] + } + + s := NewServer(50051, nil, handlers) + req := connectRequest("envoy.filters.http.ext_proc", tc.chain) + resp := s.processRequestHeaders(context.Background(), req, req.GetRequestHeaders()) + + if tc.wantStatus != 0 { + ir := resp.GetImmediateResponse() + if ir == nil { + t.Fatalf("expected an immediate response, got %v", resp) + } + if got := ir.GetStatus().GetCode(); got != tc.wantStatus { + t.Errorf("status = %v, want %v", got, tc.wantStatus) + } + if !strings.Contains(string(ir.GetBody()), "does not serve") { + t.Errorf("body = %q, want it to say the direction is not served", ir.GetBody()) + } + for d, h := range stubs { + if h.called { + t.Errorf("%s handler ran for a direction this instance does not serve", d) + } + } + return + } + + if resp.GetImmediateResponse() != nil { + t.Fatalf("unexpected immediate response: %v", resp.GetImmediateResponse()) + } + for d, h := range stubs { + if want := d == tc.wantRan; h.called != want { + t.Errorf("%s handler called = %v, want %v", d, h.called, want) + } + } + }) + } +} diff --git a/cmd/atenet/internal/router/extproc/handler.go b/cmd/atenet/internal/router/extproc/handler.go new file mode 100644 index 0000000000..a7e994e8e7 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/handler.go @@ -0,0 +1,74 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + + "github.com/agent-substrate/substrate/internal/ateattr" +) + +// Handler applies one direction's policy to a request. Implementations live in +// the ingress and egress packages; the mux never inspects what they do, only +// which direction they serve. +type Handler interface { + // Direction reports the traffic direction this handler serves. The mux keys + // its dispatch table by it. + Direction() Direction + + // HandleRequestHeaders decides what the dataplane should do with a request + // whose headers have just arrived. + // + // A returned error denies the request: a *ReqError carries the status code + // and client-safe body to answer with, anything else becomes a 500. The + // Result is read even when an error is returned, so a handler that got far + // enough to learn the metric attributes (template identity, resume outcome) + // should still fill them in. + HandleRequestHeaders(ctx context.Context, md *RequestMetadata) (Result, error) +} + +// Result is what a handler tells the mux about a request it allowed. +type Result struct { + // Response is the header mutation the dataplane applies before the request + // continues. Handlers that only authenticate return an empty CommonResponse. + Response *extprocv3.HeadersResponse + + // Target is the upstream address the request was routed to, shown on the + // /statusz page. Empty for handlers that do not pick an upstream. + Target string + + // TemplateNamespace and TemplateName identify the actor template the + // request resolved to. They are the low-cardinality attributes on the + // route-duration metric, and are empty when the direction has no template + // (or the request failed before resolving one). + TemplateNamespace string + TemplateName string + + // Resume is the actor-resume outcome, as one of the ateattr.RouterResume* + // values. Empty means "none" — the direction never resumes an actor, or the + // request never got that far. + Resume string +} + +// resume returns the resume label for the route-duration metric, defaulting an +// unset outcome to "none". +func (r Result) resume() string { + if r.Resume == "" { + return ateattr.RouterResumeNone + } + return r.Resume +} diff --git a/cmd/atenet/internal/router/extproc/metadata.go b/cmd/atenet/internal/router/extproc/metadata.go new file mode 100644 index 0000000000..7c69c58b38 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -0,0 +1,76 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "strings" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +// AuthorityHeader is the HTTP/2 pseudo-header carrying the request authority. +// Exported because the direction handlers rewrite it when they pick an +// upstream. +const AuthorityHeader = ":authority" + +// RequestMetadata is the request the mux hands to a direction handler: the +// HTTP headers the dataplane sent, flattened and lowercased, with the +// pseudo-headers every handler needs pulled out. +type RequestMetadata struct { + // Headers holds every header, keyed by lowercased name. + Headers map[string]string + Path string + Host string + Method string +} + +func NewRequestMetadata(headers []*corev3.HeaderValue) *RequestMetadata { + headersMap := make(map[string]string) + var path string + var host string + var method string + + for _, h := range headers { + k := strings.ToLower(h.Key) + val := h.Value + if val == "" && len(h.RawValue) > 0 { + val = string(h.RawValue) + } + + headersMap[k] = val + if k == ":path" { + path = val + } + if k == AuthorityHeader || k == "host" { + host = val + } + if k == ":method" { + method = val + } + } + + return &RequestMetadata{ + Headers: headersMap, + Path: path, + Host: host, + Method: method, + } +} + +// Header returns the value of a header by name, case-insensitively, or "" when +// it was not sent. +func (m *RequestMetadata) Header(name string) string { + return m.Headers[strings.ToLower(name)] +} diff --git a/cmd/atenet/internal/router/extproc_in_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go similarity index 57% rename from cmd/atenet/internal/router/extproc_in_test.go rename to cmd/atenet/internal/router/extproc/metadata_test.go index bbb7a8f9e7..5627afd0f2 100644 --- a/cmd/atenet/internal/router/extproc_in_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package extproc import ( "reflect" "testing" - "github.com/agent-substrate/substrate/internal/resources" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" ) @@ -105,78 +104,16 @@ func TestExtractMetadata(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got := newRequestMetadata(tc.headers) + got := NewRequestMetadata(tc.headers) - if !reflect.DeepEqual(got.headers, tc.wantHeaders) { - t.Errorf("extractMetadata() headersMap = %v, want %v", got.headers, tc.wantHeaders) + if !reflect.DeepEqual(got.Headers, tc.wantHeaders) { + t.Errorf("NewRequestMetadata() headersMap = %v, want %v", got.Headers, tc.wantHeaders) } - if got.path != tc.wantPath { - t.Errorf("extractMetadata() path = %v, want %v", got.path, tc.wantPath) + if got.Path != tc.wantPath { + t.Errorf("NewRequestMetadata() path = %v, want %v", got.Path, tc.wantPath) } - if got.host != tc.wantHost { - t.Errorf("extractMetadata() host = %v, want %v", got.host, tc.wantHost) - } - }) - } -} - -func TestParseActorRef(t *testing.T) { - tests := []struct { - name string - host string - want resources.ActorRef - wantErr bool - }{ - { - name: "valid host without port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:8443", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot and port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.:8080", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "missing atespace label", - host: "my-actor.actors.resources.substrate.ate.dev", - wantErr: true, - }, - { - name: "invalid suffix", - host: "my-actor.team-a.example.com", - wantErr: true, - }, - { - name: "invalid host port format", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:invalid:port", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := parseActorRef(tc.host) - if (err != nil) != tc.wantErr { - t.Errorf("parseActorRef(%q) error = %v, wantErr %v", tc.host, err, tc.wantErr) - return - } - if got != tc.want { - t.Errorf("parseActorRef(%q) = %+v, want %+v", tc.host, got, tc.want) + if got.Host != tc.wantHost { + t.Errorf("NewRequestMetadata() host = %v, want %v", got.Host, tc.wantHost) } }) } diff --git a/cmd/atenet/internal/router/extproc/metrics.go b/cmd/atenet/internal/router/extproc/metrics.go new file mode 100644 index 0000000000..e19778ce3f --- /dev/null +++ b/cmd/atenet/internal/router/extproc/metrics.go @@ -0,0 +1,111 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + "errors" + "fmt" + "time" + + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/ateattr" +) + +// ServiceName is the OpenTelemetry service name and instrumentation scope +// shared by every part of the atenet router process — the ext_proc mux here, +// the direction handlers, and the router's own servers. It lives in this +// package because it is the one package all of them already depend on. +const ServiceName = "atenet-router" + +// atenet.router.route.duration measures the latency from when the ext_proc handler receives a request +// (dataplane -> EPP) until the target worker endpoint is resolved +const routeDurationMetricName = "atenet.router.route.duration" + +// NewRouteDurationHistogram creates the atenet.router.route.duration histogram from +// the global MeterProvider. +func NewRouteDurationHistogram() (metric.Float64Histogram, error) { + h, err := otel.Meter(ServiceName).Float64Histogram( + routeDurationMetricName, + metric.WithUnit("s"), + metric.WithDescription( + "latency between Substrate router receiving a request and resolving "+ + "the target worker endpoint, excluding actor execution and response", + ), + metric.WithExplicitBucketBoundaries( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, + 0.075, 0.1, 0.15, 0.2, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30, + ), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", routeDurationMetricName, err) + } + return h, nil +} + +func (s *Server) recordRouteDuration(ctx context.Context, d time.Duration, tmplNs, tmplName, outcome, resume string) { + if s.routeDuration == nil { + return + } + s.routeDuration.Record(ctx, d.Seconds(), metric.WithAttributes( + ateattr.TemplateNamespaceKey.String(tmplNs), + ateattr.TemplateNameKey.String(tmplName), + ateattr.RouterOutcomeKey.String(outcome), + ateattr.RouterResumeKey.String(resume), + )) +} + +func classifyOutcome(err error) string { + if err == nil { + return "ok" + } + if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { + return "cancelled" + } + if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { + return "timeout" + } + switch status.Code(err) { + case codes.FailedPrecondition: + return "no_capacity" + case codes.Aborted: + return "lock_conflict" + case codes.NotFound: + return "not_found" + case codes.Unavailable: + return "unavailable" + case codes.ResourceExhausted: + return "rate_limited" + } + var re *ReqError + if errors.As(err, &re) { + switch envoy_type.StatusCode(re.StatusCode) { + case envoy_type.StatusCode_NotFound: + return "not_found" + case envoy_type.StatusCode_ServiceUnavailable: + return "no_capacity" + case envoy_type.StatusCode_GatewayTimeout: + return "timeout" + case envoy_type.StatusCode_TooManyRequests: + return "rate_limited" + } + } + return "resume_error" +} diff --git a/cmd/atenet/internal/router/extproc/metrics_test.go b/cmd/atenet/internal/router/extproc/metrics_test.go new file mode 100644 index 0000000000..c35ea705cb --- /dev/null +++ b/cmd/atenet/internal/router/extproc/metrics_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + "errors" + "testing" + "time" + + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/ateattr" +) + +func TestClassifyOutcome(t *testing.T) { + tests := []struct { + name string + err error + expected string + }{ + { + name: "nil error maps to ok", + err: nil, + expected: "ok", + }, + { + name: "context Canceled maps to cancelled", + err: context.Canceled, + expected: "cancelled", + }, + { + name: "context DeadlineExceeded maps to timeout", + err: context.DeadlineExceeded, + expected: "timeout", + }, + { + name: "FailedPrecondition gRPC code maps to no_capacity", + err: status.Error(codes.FailedPrecondition, "capacity full"), + expected: "no_capacity", + }, + { + name: "Aborted gRPC code maps to lock_conflict", + err: status.Error(codes.Aborted, "lock conflict"), + expected: "lock_conflict", + }, + { + name: "NotFound gRPC code maps to not_found", + err: status.Error(codes.NotFound, "missing"), + expected: "not_found", + }, + { + name: "Unavailable gRPC code maps to unavailable", + err: status.Error(codes.Unavailable, "control-plane down"), + expected: "unavailable", + }, + { + name: "ResourceExhausted gRPC code maps to rate_limited", + err: status.Error(codes.ResourceExhausted, "rate limit exceeded"), + expected: "rate_limited", + }, + { + name: "StatusCode_NotFound ReqError maps to not_found", + err: NewReqError(envoy_type.StatusCode_NotFound, "missing"), + expected: "not_found", + }, + { + name: "StatusCode_ServiceUnavailable ReqError maps to no_capacity", + err: NewReqError(envoy_type.StatusCode_ServiceUnavailable, "no free workers"), + expected: "no_capacity", + }, + { + name: "StatusCode_TooManyRequests ReqError maps to rate_limited", + err: NewReqError(envoy_type.StatusCode_TooManyRequests, "rate limited"), + expected: "rate_limited", + }, + { + name: "Unknown error maps to resume_error", + err: errors.New("internal storage glitch"), + expected: "resume_error", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := classifyOutcome(tc.err); got != tc.expected { + t.Errorf("classifyOutcome(%v) = %q, want %q", tc.err, got, tc.expected) + } + }) + } +} + +func TestRecordRouteDuration_Attributes(t *testing.T) { + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + h, err := mp.Meter("atenet-router").Float64Histogram(routeDurationMetricName) + if err != nil { + t.Fatalf("failed to create histogram: %v", err) + } + + s := NewServer(50051, h, nil) + s.recordRouteDuration(context.Background(), 10*time.Millisecond, "team-a-ns", "tmpl-a", classifyOutcome(nil), ateattr.RouterResumeTriggered) + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect failed: %v", err) + } + + dp := rm.ScopeMetrics[0].Metrics[0].Data.(metricdata.Histogram[float64]).DataPoints[0] + wantAttrs := map[string]string{ + "ate.template.namespace": "team-a-ns", + "ate.template.name": "tmpl-a", + "ate.router.outcome": "ok", + "ate.router.resume": "triggered", + } + + for k, want := range wantAttrs { + val, exists := dp.Attributes.Value(attribute.Key(k)) + if !exists { + t.Errorf("missing metric attribute %q", k) + } else if val.AsString() != want { + t.Errorf("attribute %q = %q, want %q", k, val.AsString(), want) + } + } +} diff --git a/cmd/atenet/internal/router/extproc/record.go b/cmd/atenet/internal/router/extproc/record.go new file mode 100644 index 0000000000..51c6b8774e --- /dev/null +++ b/cmd/atenet/internal/router/extproc/record.go @@ -0,0 +1,117 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "strings" + "sync" + "time" +) + +type RecordedQuery struct { + Timestamp time.Time `json:"timestamp"` + Client string `json:"client"` + Host string `json:"host"` + Path string `json:"path"` + Method string `json:"method"` + Action string `json:"action"` + Target string `json:"target"` + Duration time.Duration `json:"duration"` +} + +// QueryRecorder is the fixed-size ring of recently processed requests behind +// the router's /statusz page. +type QueryRecorder struct { + mu sync.RWMutex + queries []RecordedQuery + size int + index int +} + +func NewQueryRecorder(size int) *QueryRecorder { + return &QueryRecorder{ + queries: make([]RecordedQuery, 0, size), + size: size, + } +} + +func (qr *QueryRecorder) Add(q RecordedQuery) { + if qr == nil { + return + } + + qr.mu.Lock() + defer qr.mu.Unlock() + + if len(qr.queries) < qr.size { + qr.queries = append(qr.queries, q) + } else { + qr.queries[qr.index] = q + qr.index = (qr.index + 1) % qr.size + } +} + +func (qr *QueryRecorder) Get() []RecordedQuery { + if qr == nil { + return nil + } + + qr.mu.RLock() + defer qr.mu.RUnlock() + + n := len(qr.queries) + if n == 0 { + return nil + } + + res := make([]RecordedQuery, n) + if n < qr.size { + for i := 0; i < n; i++ { + res[i] = qr.queries[n-1-i] + } + } else { + for i := 0; i < n; i++ { + pos := (qr.index - 1 - i + n) % n + res[i] = qr.queries[pos] + } + } + + return res +} + +// redactPath drops the query string, which may carry credentials (CWE-598). +func redactPath(path string) string { + p, _, _ := strings.Cut(path, "?") + return p +} + +func (qr *QueryRecorder) AddRouterRequest( + start time.Time, + duration time.Duration, + action, + target string, + m *RequestMetadata, +) { + qr.Add(RecordedQuery{ + Timestamp: start, + Client: m.Headers[AuthorityHeader], + Host: m.Host, + Path: redactPath(m.Path), + Method: m.Headers[":method"], + Action: action, + Target: target, + Duration: duration, + }) +} diff --git a/cmd/atenet/internal/router/extproc_in.go b/cmd/atenet/internal/router/extproc_in.go deleted file mode 100644 index 6ab2d2a6d7..0000000000 --- a/cmd/atenet/internal/router/extproc_in.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "net" - "strings" - - "github.com/agent-substrate/substrate/internal/resources" - corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" -) - -const authorityHeader = ":authority" - -type requestMetadata struct { - headers map[string]string - path string - host string -} - -func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata { - headersMap := make(map[string]string) - var path string - var host string - - for _, h := range headers { - k := strings.ToLower(h.Key) - val := h.Value - if val == "" && len(h.RawValue) > 0 { - val = string(h.RawValue) - } - - headersMap[k] = val - if k == ":path" { - path = val - } - if k == authorityHeader || k == "host" { - host = val - } - } - - return &requestMetadata{ - headers: headersMap, - path: path, - host: host, - } -} - -// parseActorRef extracts the actor an incoming request is addressed to from its -// Host/:authority, which has the form -// "..actors.resources.substrate.ate.dev" (optionally with a -// port). The atespace is part of the name because an actor name is only unique -// within its atespace. -func parseActorRef(host string) (resources.ActorRef, error) { - if strings.Contains(host, ":") { - h, _, err := net.SplitHostPort(host) - if err != nil { - return resources.ActorRef{}, err - } - host = h - } - return resources.ParseActorDNSName(host) -} diff --git a/cmd/atenet/internal/router/extproc_out.go b/cmd/atenet/internal/router/extproc_out.go deleted file mode 100644 index a6759daf6b..0000000000 --- a/cmd/atenet/internal/router/extproc_out.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "github.com/agent-substrate/substrate/internal/atunnel" - corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" - envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" -) - -// reqError carries an HTTP-mappable status code and a client-safe message. -// The underlying cause (if any) is preserved via Unwrap so logs can inspect -// the full chain without leaking server-side detail into the response body. -type reqError struct { - msg string - cause error - statusCode int -} - -func (e *reqError) Error() string { return e.msg } -func (e *reqError) Unwrap() error { return e.cause } - -// addOriginalDstMutation sets the header the ORIGINAL_DST cluster reads to pick -// the upstream address (the worker atunnel IP:443). Unlike an :authority -// rewrite it leaves the request Host intact, so atunnel still sees the actor -// DNS name and can authorize the active actor. -// -// Nothing strips this header from the incoming request, so overwrite rather -// than append: a client-supplied value must never influence the address Envoy -// dials. ext_proc mutations already default to replace, but the default is -// split across the deprecated append field and append_action — pin it. -func addOriginalDstMutation(dst string, mut *extproc.HeaderMutation) { - mut.SetHeaders = append(mut.SetHeaders, - &corev3.HeaderValueOption{ - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - Header: &corev3.HeaderValue{ - Key: OriginalDstHeader, - RawValue: []byte(dst), - }, - }, - ) -} - -// addRoutingMutations overwrites all routing metadata derived from the -// control-plane result. Envoy dials OriginalDstHeader while preserving -// :authority. Agentgateway v1.4.1's static dynamic backend instead dials the -// request :authority, so that mode rewrites it to the worker atunnel address. -// OriginalHostHeader lets atunnel restore and authorize the actor authority. -func addRoutingMutations(dst, actorHost string, routeViaAuthority bool, mut *extproc.HeaderMutation) { - addOriginalDstMutation(dst, mut) - mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - Header: &corev3.HeaderValue{ - Key: atunnel.OriginalHostHeader, - RawValue: []byte(actorHost), - }, - }) - if routeViaAuthority { - mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - Header: &corev3.HeaderValue{ - Key: authorityHeader, - RawValue: []byte(dst), - }, - }) - } -} - -func immediateResponse(statusCode envoy_type.StatusCode, message string) *extproc.ProcessingResponse { - return &extproc.ProcessingResponse{ - Response: &extproc.ProcessingResponse_ImmediateResponse{ - ImmediateResponse: &extproc.ImmediateResponse{ - Status: &envoy_type.HttpStatus{ - Code: statusCode, - }, - Body: []byte(message), - Headers: &extproc.HeaderMutation{ - SetHeaders: []*corev3.HeaderValueOption{ - { - // Using RawValues instead of Value: newer versions of Envoy - // drop Value and use RawValue - Header: &corev3.HeaderValue{ - Key: "content-type", - RawValue: []byte("text/plain"), - }, - }, - }, - }, - }, - }, - } -} diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index 49ee002a0d..8f911f696a 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -157,6 +157,14 @@ func updateComponentHealth(health *ComponentHealth, healthy bool, msg string, ch } func (rh *routerHealth) checkDataplane(ctx context.Context) (bool, string) { + // The dataplane this polls is the *ingress* proxy sharing the router's pod. + // The egress gateway is a separate, statically configured proxy on its own + // admin port; an egress-only router has none beside it, and probing this + // address would report a permanently unhealthy dependency. + if !rh.cfg.Mode.ServesIngress() { + return true, "Skipped (egress mode)" + } + timeoutCtx, cancel := context.WithTimeout(ctx, dependencyHealthCheckTimeout) defer cancel() diff --git a/cmd/atenet/internal/router/health_test.go b/cmd/atenet/internal/router/health_test.go index e1b4c03932..dd594052b3 100644 --- a/cmd/atenet/internal/router/health_test.go +++ b/cmd/atenet/internal/router/health_test.go @@ -16,6 +16,7 @@ package router import ( "context" + "errors" "io" "net/http" "net/http/httptest" @@ -112,6 +113,25 @@ func TestCheckDataplane(t *testing.T) { } } +// An egress-only router has no ingress dataplane beside it, so probing the +// ingress proxy's admin port would report a dependency that is permanently +// down. +func TestCheckDataplaneSkippedInEgressMode(t *testing.T) { + rh := newRouterHealth(time.Second, nil, nil, routerConfig{Mode: ModeEgress}) + rh.dataplaneClient = &http.Client{Transport: healthRoundTripFunc(func(*http.Request) (*http.Response, error) { + t.Error("checkDataplane dialed the ingress dataplane admin port in egress mode") + return nil, errors.New("unexpected request") + })} + + healthy, msg := rh.checkDataplane(context.Background()) + if !healthy { + t.Errorf("checkDataplane healthy = false, want true (skipped)") + } + if !strings.Contains(msg, "Skipped") { + t.Errorf("checkDataplane message = %q, want it to report the check was skipped", msg) + } +} + func TestCheckK8sTimesOut(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { <-req.Context().Done() diff --git a/cmd/atenet/internal/router/ingress/actorref_test.go b/cmd/atenet/internal/router/ingress/actorref_test.go new file mode 100644 index 0000000000..a87639f243 --- /dev/null +++ b/cmd/atenet/internal/router/ingress/actorref_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import ( + "testing" + + "github.com/agent-substrate/substrate/internal/resources" +) + +func TestParseActorRef(t *testing.T) { + tests := []struct { + name string + host string + want resources.ActorRef + wantErr bool + }{ + { + name: "valid host without port", + host: "my-actor.team-a.actors.resources.substrate.ate.dev", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "valid host with port", + host: "my-actor.team-a.actors.resources.substrate.ate.dev:8443", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "valid host with trailing dot", + host: "my-actor.team-a.actors.resources.substrate.ate.dev.", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "valid host with trailing dot and port", + host: "my-actor.team-a.actors.resources.substrate.ate.dev.:8080", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "missing atespace label", + host: "my-actor.actors.resources.substrate.ate.dev", + wantErr: true, + }, + { + name: "invalid suffix", + host: "my-actor.team-a.example.com", + wantErr: true, + }, + { + name: "invalid host port format", + host: "my-actor.team-a.actors.resources.substrate.ate.dev:invalid:port", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseActorRef(tc.host) + if (err != nil) != tc.wantErr { + t.Errorf("parseActorRef(%q) error = %v, wantErr %v", tc.host, err, tc.wantErr) + return + } + if got != tc.want { + t.Errorf("parseActorRef(%q) = %+v, want %+v", tc.host, got, tc.want) + } + }) + } +} diff --git a/cmd/atenet/internal/router/errors.go b/cmd/atenet/internal/router/ingress/errors.go similarity index 60% rename from cmd/atenet/internal/router/errors.go rename to cmd/atenet/internal/router/ingress/errors.go index 2511487471..5c08f57a21 100644 --- a/cmd/atenet/internal/router/errors.go +++ b/cmd/atenet/internal/router/ingress/errors.go @@ -12,41 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" "errors" "fmt" - "github.com/agent-substrate/substrate/internal/resources" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" -) -// newReqError builds a reqError whose body is the formatted message and no -// wrapped cause. Set the cause field directly when one is available. -func newReqError(code envoy_type.StatusCode, format string, args ...any) error { - return &reqError{ - msg: fmt.Sprintf(format, args...), - statusCode: int(code), - } -} + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/resources" +) -// actorNotFoundErr returns a 404 reqError identifying the missing actor. +// actorNotFoundErr returns a 404 denial identifying the missing actor. func actorNotFoundErr(actorRef resources.ActorRef) error { - return newReqError(envoy_type.StatusCode_NotFound, "actor %s not found", actorRef) + return extproc.NewReqError(envoy_type.StatusCode_NotFound, "actor %s not found", actorRef) } -// invalidHostErr returns a 404 reqError explaining why the request host was +// invalidHostErr returns a 404 denial explaining why the request host was // rejected. The cause is preserved for log inspection via Unwrap. func invalidHostErr(host string, cause error) error { - return &reqError{ - msg: fmt.Sprintf("invalid host %q: %v", host, cause), - cause: cause, - statusCode: int(envoy_type.StatusCode_NotFound), - } + return extproc.WrapReqError(envoy_type.StatusCode_NotFound, cause, "invalid host %q: %v", host, cause) } // statusDescription returns the gRPC status description of err, unwrapping @@ -62,15 +51,15 @@ func statusDescription(err error) string { return status.Convert(err).Message() } -// parkingFullErr returns a 503 reqError signaling that the router's parking lot +// parkingFullErr returns a 503 denial signaling that the router's parking lot // is at capacity, so the request was shed without waiting. Clients should retry. func parkingFullErr(actorID string) error { - return newReqError(envoy_type.StatusCode_ServiceUnavailable, + return extproc.NewReqError(envoy_type.StatusCode_ServiceUnavailable, "actor %q unavailable: router at capacity", actorID) } // mapResumeError translates an ActorResumer error into a client-facing -// reqError. It maps gRPC status codes to appropriate HTTP status codes and +// denial. It maps gRPC status codes to appropriate HTTP status codes and // short, human-readable bodies. The original error is preserved via Unwrap // so callers can still inspect it via errors.Is / errors.As when logging. // @@ -81,7 +70,7 @@ func mapResumeError(actorRef resources.ActorRef, err error) error { return nil } - re := &reqError{cause: err} + re := &extproc.ReqError{Cause: err} // Bare context sentinels reach here when the request's own context ends // (client disconnect or stream deadline) — status.Code would classify them @@ -90,50 +79,50 @@ func mapResumeError(actorRef resources.ActorRef, err error) error { // already dead, so the code is observability-only; Envoy's StatusCode enum // has no 499 ("client closed request"), so 408 is the nearest defined code. if errors.Is(err, context.Canceled) { - re.statusCode = int(envoy_type.StatusCode_RequestTimeout) - re.msg = fmt.Sprintf("request for actor %s canceled by client", actorRef) + re.StatusCode = int(envoy_type.StatusCode_RequestTimeout) + re.Msg = fmt.Sprintf("request for actor %s canceled by client", actorRef) return re } if errors.Is(err, context.DeadlineExceeded) { - re.statusCode = int(envoy_type.StatusCode_GatewayTimeout) - re.msg = fmt.Sprintf("actor %s request timed out", actorRef) + re.StatusCode = int(envoy_type.StatusCode_GatewayTimeout) + re.Msg = fmt.Sprintf("actor %s request timed out", actorRef) return re } switch status.Code(err) { case codes.NotFound: - re.statusCode = int(envoy_type.StatusCode_NotFound) - re.msg = fmt.Sprintf("actor %s not found", actorRef) + re.StatusCode = int(envoy_type.StatusCode_NotFound) + re.Msg = fmt.Sprintf("actor %s not found", actorRef) case codes.FailedPrecondition: // Preserve the gRPC description for FailedPrecondition and Aborted: // they carry actionable client-facing context (e.g. "no free workers // available", "another operation is in progress for this actor") and // are not security-sensitive. - re.statusCode = int(envoy_type.StatusCode_ServiceUnavailable) - re.msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) + re.StatusCode = int(envoy_type.StatusCode_ServiceUnavailable) + re.Msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) case codes.Aborted: // A concurrency conflict that outlived its retries (e.g. a park budget // spent entirely on Aborted). Retryable by the client, hence 503. - re.statusCode = int(envoy_type.StatusCode_ServiceUnavailable) - re.msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) + re.StatusCode = int(envoy_type.StatusCode_ServiceUnavailable) + re.Msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) case codes.Unavailable: - re.statusCode = int(envoy_type.StatusCode_ServiceUnavailable) - re.msg = fmt.Sprintf("actor %s unavailable", actorRef) + re.StatusCode = int(envoy_type.StatusCode_ServiceUnavailable) + re.Msg = fmt.Sprintf("actor %s unavailable", actorRef) case codes.DeadlineExceeded: - re.statusCode = int(envoy_type.StatusCode_GatewayTimeout) - re.msg = fmt.Sprintf("actor %s request timed out", actorRef) + re.StatusCode = int(envoy_type.StatusCode_GatewayTimeout) + re.Msg = fmt.Sprintf("actor %s request timed out", actorRef) case codes.PermissionDenied: - re.statusCode = int(envoy_type.StatusCode_Forbidden) - re.msg = fmt.Sprintf("actor %s access denied", actorRef) + re.StatusCode = int(envoy_type.StatusCode_Forbidden) + re.Msg = fmt.Sprintf("actor %s access denied", actorRef) case codes.Unauthenticated: - re.statusCode = int(envoy_type.StatusCode_Unauthorized) - re.msg = fmt.Sprintf("actor %s authentication required", actorRef) + re.StatusCode = int(envoy_type.StatusCode_Unauthorized) + re.Msg = fmt.Sprintf("actor %s authentication required", actorRef) case codes.ResourceExhausted: - re.statusCode = int(envoy_type.StatusCode_TooManyRequests) - re.msg = fmt.Sprintf("actor %s rate limited", actorRef) + re.StatusCode = int(envoy_type.StatusCode_TooManyRequests) + re.Msg = fmt.Sprintf("actor %s rate limited", actorRef) default: - re.statusCode = int(envoy_type.StatusCode_InternalServerError) - re.msg = fmt.Sprintf("error resuming actor %s", actorRef) + re.StatusCode = int(envoy_type.StatusCode_InternalServerError) + re.Msg = fmt.Sprintf("error resuming actor %s", actorRef) } return re } diff --git a/cmd/atenet/internal/router/errors_test.go b/cmd/atenet/internal/router/ingress/errors_test.go similarity index 72% rename from cmd/atenet/internal/router/errors_test.go rename to cmd/atenet/internal/router/ingress/errors_test.go index 3de804c7a1..0e82f4af9b 100644 --- a/cmd/atenet/internal/router/errors_test.go +++ b/cmd/atenet/internal/router/ingress/errors_test.go @@ -12,48 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" "errors" "testing" - "github.com/agent-substrate/substrate/internal/resources" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" -) -func TestNewReqError(t *testing.T) { - t.Parallel() - - err := newReqError(envoy_type.StatusCode_BadRequest, "actor %q is %s", "abc", "bad") - if err == nil { - t.Fatal("newReqError returned nil") - } - var reqErr *reqError - if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) - } - if reqErr.statusCode != int(envoy_type.StatusCode_BadRequest) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, envoy_type.StatusCode_BadRequest) - } - if got, want := err.Error(), `actor "abc" is bad`; got != want { - t.Errorf("Error() = %q, want %q", got, want) - } -} + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/resources" +) func TestActorNotFoundErr(t *testing.T) { t.Parallel() err := actorNotFoundErr(resources.ActorRef{Atespace: "team-a", Name: "ctr6"}) - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) } - if reqErr.statusCode != int(envoy_type.StatusCode_NotFound) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, envoy_type.StatusCode_NotFound) + if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) } if got, want := err.Error(), `actor team-a/ctr6 not found`; got != want { t.Errorf("Error() = %q, want %q", got, want) @@ -66,12 +49,12 @@ func TestInvalidHostErr(t *testing.T) { cause := errors.New("missing suffix") err := invalidHostErr("foo.example.com", cause) - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) } - if reqErr.statusCode != int(envoy_type.StatusCode_NotFound) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, envoy_type.StatusCode_NotFound) + if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) } if got, want := err.Error(), `invalid host "foo.example.com": missing suffix`; got != want { t.Errorf("Error() = %q, want %q", got, want) @@ -186,12 +169,12 @@ func TestMapResumeError(t *testing.T) { if got == nil { t.Fatal("mapResumeError returned nil") } - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(got, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", got) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", got) } - if reqErr.statusCode != int(tc.wantCode) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, tc.wantCode) + if reqErr.StatusCode != int(tc.wantCode) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, tc.wantCode) } if got.Error() != tc.wantBody { t.Errorf("Error() = %q, want %q", got.Error(), tc.wantBody) @@ -213,35 +196,14 @@ func TestMapResumeError_NilError(t *testing.T) { } } -// Ensures mapResumeError result satisfies the reqError contract so the -// existing handleRequestHeaders branch (errors.As(err, &reqErr)) keeps working. +// Ensures mapResumeError result satisfies the extproc.ReqError contract so the +// mux's errors.As(err, &reqErr) branch keeps mapping it to the right status. func TestMapResumeError_IsReqError(t *testing.T) { t.Parallel() err := mapResumeError(resources.ActorRef{Atespace: "team-a", Name: "x"}, status.Error(codes.NotFound, "x")) - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) - } -} - -// TestImmediateResponseHeaderEncoding pins the RawValue encoding: Envoy drops -// plain Value in ext_proc header mutations, so a Value-encoded header reaches -// the client with an empty value (found live — content-type on every immediate -// response had been arriving empty). -func TestImmediateResponseHeaderEncoding(t *testing.T) { - t.Parallel() - - resp := immediateResponse(envoy_type.StatusCode_ServiceUnavailable, "body") - set := resp.GetImmediateResponse().GetHeaders().GetSetHeaders() - if len(set) != 1 { - t.Fatalf("SetHeaders count = %d, want 1", len(set)) - } - h := set[0].GetHeader() - if h.GetKey() != "content-type" || string(h.GetRawValue()) != "text/plain" { - t.Errorf("header = %q:%q (RawValue), want content-type:text/plain", h.GetKey(), h.GetRawValue()) - } - if h.GetValue() != "" { - t.Errorf("header uses Value (%q); must use RawValue only", h.GetValue()) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) } } diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go new file mode 100644 index 0000000000..a784696e67 --- /dev/null +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -0,0 +1,211 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ingress implements the ext_proc handler for traffic arriving at the +// ingress gateway: it resolves the actor a request is addressed to, resumes it +// through the control plane (parking the request while the worker pool is +// saturated), and points the dataplane at the worker that ends up hosting it. +// +// Everything reaching this handler is unauthenticated client input. The +// opposite trust model — an actor identity carried by a CA-signed client +// certificate — belongs to the sibling egress package, and the two are kept +// apart deliberately. +package ingress + +import ( + "context" + "log/slog" + "net" + "strings" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// OriginalDstHeader carries the resolved worker atunnel address (IP:443) to +// the ORIGINAL_DST cluster. It is this handler's contract with the ingress +// Envoy configuration the xDS server generates. +const OriginalDstHeader = "x-ate-original-dst" + +// Handler routes ingress requests to the worker hosting their actor. +type Handler struct { + resumer *ActorResumer + parking *parkingLot + // routeViaAuthority rewrites :authority to the worker atunnel address for + // data planes that dial it rather than OriginalDstHeader. See + // addRoutingMutations. + routeViaAuthority bool +} + +func New(apiClient ateapipb.ControlClient, parkCfg ParkedRequestConfig, parkMetrics *ParkingMetrics, routeViaAuthority bool) *Handler { + return &Handler{ + resumer: NewActorResumer(apiClient, withParking(parkCfg)), + parking: newParkingLot(parkCfg, parkMetrics), + routeViaAuthority: routeViaAuthority, + } +} + +func (h *Handler) Direction() extproc.Direction { return extproc.DirectionIngress } + +// ParkingStatus returns a snapshot of the parking lot for the /statusz page. +func (h *Handler) ParkingStatus() ParkingStatus { return h.parking.status() } + +func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestMetadata) (extproc.Result, error) { + slog.InfoContext(ctx, "Request", slog.String("host", md.Host)) + + // The dataplane doesn't propagate trace context into the ext_proc gRPC + // stream's metadata — the per-request traceparent arrives in the + // HTTP headers carried inside the ProcessingRequest payload. Extract + // from there so our span links to the gateway's ingress span. + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(md.Headers)) + ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") + defer span.End() + + actorRef, err := parseActorRef(md.Host) + if err != nil { + // Host is invalid, respond with 404. + return extproc.Result{}, invalidHostErr(md.Host, err) + } + + // Admit the request to the parking lot before resuming. While resume is + // in-flight the request occupies a slot; if the actor's worker pool is + // momentarily saturated the resumer parks (retries) here rather than failing + // fast. A full lot sheds the request immediately so the router applies + // backpressure instead of queueing without bound. + release, ok := h.parking.enter(ctx) + if !ok { + return extproc.Result{}, parkingFullErr(actorRef.String()) + } + + slog.InfoContext(ctx, "ResumeActor", slog.Any("actor", actorRef)) + actor, resumeOutcome, err := h.resumer.ResumeActor(ctx, actorRef) + release(parkOutcomeFor(err)) + if err != nil { + return extproc.Result{Resume: string(resumeOutcome)}, mapResumeError(actorRef, err) + } + + // Actor template identity, used as low-cardinality route-latency metric + // attributes. + res := extproc.Result{ + TemplateNamespace: actor.GetActorTemplateNamespace(), + TemplateName: actor.GetActorTemplateName(), + Resume: string(resumeOutcome), + } + + workerIP := actor.GetWorkerAssignment().GetWorkerPodIp() + slog.InfoContext(ctx, "ResumeActor result", + slog.Any("actor", actorRef), + slog.String("status", actor.GetStatus().String()), + slog.String("workerIP", workerIP)) + + if ip := net.ParseIP(workerIP); ip == nil { + return res, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, + "actor %s routing failed", actorRef) + } + + // The actor is reached through the in-worker atunnel ingress server, which + // listens on :443 (mTLS) and forwards to the actor's :80. The worker no + // longer DNATs pod-IP:80 to the actor, so the router dials :443 and the + // ORIGINAL_DST cluster's upstream TLS context presents the router's + // podidentity client cert (see buildOriginalDstCluster and + // buildUpstreamTransportSocket). + // TODO(bowei) -- handle more than port 80 on the actor. + targetAddr := net.JoinHostPort(workerIP, "443") + + slog.InfoContext(ctx, "Route ok", slog.Any("actor", actorRef), slog.String("targetAddr", targetAddr)) + + // Route by telling the ORIGINAL_DST cluster which worker atunnel address to + // dial, without touching :authority — atunnel authorizes the actor by the + // original Host (actor DNS name). + mutation := &extprocv3.HeaderMutation{} + addRoutingMutations(targetAddr, md.Host, h.routeViaAuthority, mutation) + + res.Target = targetAddr + res.Response = &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{ + HeaderMutation: mutation, + }, + } + return res, nil +} + +// parseActorRef extracts the actor an incoming request is addressed to from its +// Host/:authority, which has the form +// "..actors.resources.substrate.ate.dev" (optionally with a +// port). The atespace is part of the name because an actor name is only unique +// within its atespace. +func parseActorRef(host string) (resources.ActorRef, error) { + if strings.Contains(host, ":") { + h, _, err := net.SplitHostPort(host) + if err != nil { + return resources.ActorRef{}, err + } + host = h + } + return resources.ParseActorDNSName(host) +} + +// addOriginalDstMutation sets the header the ORIGINAL_DST cluster reads to pick +// the upstream address (the worker atunnel IP:443). Unlike an :authority +// rewrite it leaves the request Host intact, so atunnel still sees the actor +// DNS name and can authorize the active actor. +// +// Nothing strips this header from the incoming request, so overwrite rather +// than append: a client-supplied value must never influence the address Envoy +// dials. ext_proc mutations already default to replace, but the default is +// split across the deprecated append field and append_action — pin it. +func addOriginalDstMutation(dst string, mut *extprocv3.HeaderMutation) { + mut.SetHeaders = append(mut.SetHeaders, + &corev3.HeaderValueOption{ + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + Header: &corev3.HeaderValue{ + Key: OriginalDstHeader, + RawValue: []byte(dst), + }, + }, + ) +} + +// addRoutingMutations overwrites all routing metadata derived from the +// control-plane result. Envoy dials OriginalDstHeader while preserving +// :authority. Agentgateway v1.4.1's static dynamic backend instead dials the +// request :authority, so that mode rewrites it to the worker atunnel address. +// OriginalHostHeader lets atunnel restore and authorize the actor authority. +func addRoutingMutations(dst, actorHost string, routeViaAuthority bool, mut *extprocv3.HeaderMutation) { + addOriginalDstMutation(dst, mut) + mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + Header: &corev3.HeaderValue{ + Key: atunnel.OriginalHostHeader, + RawValue: []byte(actorHost), + }, + }) + if routeViaAuthority { + mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + Header: &corev3.HeaderValue{ + Key: extproc.AuthorityHeader, + RawValue: []byte(dst), + }, + }) + } +} diff --git a/cmd/atenet/internal/router/extproc_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go similarity index 61% rename from cmd/atenet/internal/router/extproc_test.go rename to cmd/atenet/internal/router/ingress/ingress_test.go index 5173b40dbd..1620a1060e 100644 --- a/cmd/atenet/internal/router/extproc_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "bytes" @@ -24,17 +24,16 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "go.opentelemetry.io/otel/attribute" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) type mockClient struct { @@ -46,6 +45,12 @@ func (m *mockClient) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRe return m.resumeFn(ctx, in, opts...) } +// requestMetadata builds the metadata the ext_proc mux would hand the handler +// for a request with these headers. +func requestMetadata(headers ...*corev3.HeaderValue) *extproc.RequestMetadata { + return extproc.NewRequestMetadata(headers) +} + func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" const secret = "do-not-log-me" @@ -55,25 +60,21 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) t.Cleanup(func() { slog.SetDefault(prev) }) - s := NewExtProcServer(50051, &mockClient{ + h := New(&mockClient{ resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}}, nil }, - }, nil, ParkedRequestConfig{}, nil, false) + }, ParkedRequestConfig{}, nil, false) - reqHeaders := &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":path", Value: "/api/v1/reset?token=" + secret}, - {Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, - {Key: ":method", Value: "POST"}, - {Key: "authorization", Value: "Bearer " + secret}, - {Key: "cookie", Value: "session=" + secret}, - }, - }, - } + md := requestMetadata( + &corev3.HeaderValue{Key: ":path", Value: "/api/v1/reset?token=" + secret}, + &corev3.HeaderValue{Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, + &corev3.HeaderValue{Key: ":method", Value: "POST"}, + &corev3.HeaderValue{Key: "authorization", Value: "Bearer " + secret}, + &corev3.HeaderValue{Key: "cookie", Value: "session=" + secret}, + ) - _, metadata, target, _, _, _, err := s.handleRequestHeaders(context.Background(), reqHeaders) + res, err := h.HandleRequestHeaders(context.Background(), md) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -86,15 +87,18 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { t.Errorf("router log missing actor/host routing context: %s", out) } - s.recorder.AddRouterRequest(time.Now(), time.Millisecond, "Route ok", target, metadata) - for _, q := range s.recorder.Get() { + // The mux records every handled request on the status page; the metadata the + // handler was given must not carry the secret into it either. + rec := extproc.NewQueryRecorder(10) + rec.AddRouterRequest(time.Now(), time.Millisecond, "Route ok", res.Target, md) + for _, q := range rec.Get() { if blob, _ := json.Marshal(q); strings.Contains(string(blob), secret) { t.Errorf("recorder/statusz retained sensitive value: %s", blob) } } } -func TestExtProcHeadersEvaluation(t *testing.T) { +func TestHandleRequestHeaders(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" tests := []struct { @@ -195,20 +199,17 @@ func TestExtProcHeadersEvaluation(t *testing.T) { // Parking disabled: these cases assert fail-fast mapping of resume // errors (e.g. FailedPrecondition -> immediate 503). Parking behavior - // is covered separately in TestExtProc_ParkingLotFull and resumer_test.go. - s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{}, nil, false) + // is covered separately in TestHandleRequestHeaders_ParkingLotFull and + // resumer_test.go. + h := New(clientMock, ParkedRequestConfig{}, nil, false) - reqHeaders := &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":path", Value: "/v1/actors/invoke"}, - {Key: ":authority", Value: tc.authority}, - {Key: ":method", Value: "POST"}, - }, - }, - } + md := requestMetadata( + &corev3.HeaderValue{Key: ":path", Value: "/v1/actors/invoke"}, + &corev3.HeaderValue{Key: ":authority", Value: tc.authority}, + &corev3.HeaderValue{Key: ":method", Value: "POST"}, + ) - res, metadata, target, _, _, _, err := s.handleRequestHeaders(context.Background(), reqHeaders) + res, err := h.HandleRequestHeaders(context.Background(), md) if tc.expectErr { if err == nil { t.Fatalf("expected error but got nil") @@ -216,11 +217,11 @@ func TestExtProcHeadersEvaluation(t *testing.T) { if tc.expectedErrStr != "" && err.Error() != tc.expectedErrStr { t.Errorf("client body mismatch:\n got: %q\n want: %q", err.Error(), tc.expectedErrStr) } - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("expected *reqError, got %T (%v)", err, err) + t.Fatalf("expected *extproc.ReqError, got %T (%v)", err, err) } - if got, want := reqErr.statusCode, int(tc.expectedStatus); got != want { + if got, want := reqErr.StatusCode, int(tc.expectedStatus); got != want { t.Errorf("HTTP status code = %d, want %d", got, want) } if tc.resumeErr != nil && !errors.Is(err, tc.resumeErr) { @@ -232,11 +233,11 @@ func TestExtProcHeadersEvaluation(t *testing.T) { if err != nil { t.Fatalf("ext_proc processing error: %v", err) } - if target != tc.expectedTarget { - t.Errorf("expected target %q, got %q", tc.expectedTarget, target) + if res.Target != tc.expectedTarget { + t.Errorf("expected target %q, got %q", tc.expectedTarget, res.Target) } - mutation := res.Response.GetHeaderMutation() + mutation := res.Response.GetResponse().GetHeaderMutation() if len(mutation.GetSetHeaders()) != 2 { t.Fatalf("expected exactly two header options, found: %v", mutation.GetSetHeaders()) } @@ -252,19 +253,13 @@ func TestExtProcHeadersEvaluation(t *testing.T) { t.Errorf("original host mutation = %q, want %q", got, tc.authority) } - // Confirm that query logs recorded metric trace details - s.recorder.AddRouterRequest(time.Now(), 10*time.Millisecond, "Route ok", tc.expectedTarget, metadata) - queries := s.recorder.Get() - if len(queries) != 1 { - t.Errorf("expected query trace entries, got: %v", queries) - } }) } } -// TestExtProc_ParkingLotFull verifies that when the parking lot is at capacity +// TestHandleRequestHeaders_ParkingLotFull verifies that when the parking lot is at capacity // the request is shed with a 503 before any resume is attempted. -func TestExtProc_ParkingLotFull(t *testing.T) { +func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" var resumeCalled bool @@ -277,31 +272,27 @@ func TestExtProc_ParkingLotFull(t *testing.T) { // A 1-slot lot with the slot already occupied deterministically simulates a // full lot without needing a concurrent in-flight request. - s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false) - release, ok := s.parking.enter(context.Background()) + h := New(clientMock, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false) + release, ok := h.parking.enter(context.Background()) if !ok { t.Fatal("priming enter should be admitted") } defer release(parkOutcomeServed) - reqHeaders := &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, - }, - }, - } + md := requestMetadata( + &corev3.HeaderValue{Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, + ) - _, _, _, _, _, _, err := s.handleRequestHeaders(context.Background(), reqHeaders) + _, err := h.HandleRequestHeaders(context.Background(), md) if err == nil { t.Fatal("expected error when parking lot is full") } - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("expected *reqError, got %T (%v)", err, err) + t.Fatalf("expected *extproc.ReqError, got %T (%v)", err, err) } - if reqErr.statusCode != int(envoy_type.StatusCode_ServiceUnavailable) { - t.Errorf("status code = %d, want %d (503)", reqErr.statusCode, envoy_type.StatusCode_ServiceUnavailable) + if reqErr.StatusCode != int(envoy_type.StatusCode_ServiceUnavailable) { + t.Errorf("status code = %d, want %d (503)", reqErr.StatusCode, envoy_type.StatusCode_ServiceUnavailable) } if !strings.Contains(reqErr.Error(), "router at capacity") { t.Errorf("error body = %q, want it to mention capacity", reqErr.Error()) @@ -311,117 +302,6 @@ func TestExtProc_ParkingLotFull(t *testing.T) { } } -func TestClassifyOutcome(t *testing.T) { - tests := []struct { - name string - err error - expected string - }{ - { - name: "nil error maps to ok", - err: nil, - expected: "ok", - }, - { - name: "context Canceled maps to cancelled", - err: context.Canceled, - expected: "cancelled", - }, - { - name: "context DeadlineExceeded maps to timeout", - err: context.DeadlineExceeded, - expected: "timeout", - }, - { - name: "FailedPrecondition gRPC code maps to no_capacity", - err: status.Error(codes.FailedPrecondition, "capacity full"), - expected: "no_capacity", - }, - { - name: "Aborted gRPC code maps to lock_conflict", - err: status.Error(codes.Aborted, "lock conflict"), - expected: "lock_conflict", - }, - { - name: "NotFound gRPC code maps to not_found", - err: status.Error(codes.NotFound, "missing"), - expected: "not_found", - }, - { - name: "Unavailable gRPC code maps to unavailable", - err: status.Error(codes.Unavailable, "control-plane down"), - expected: "unavailable", - }, - { - name: "ResourceExhausted gRPC code maps to rate_limited", - err: status.Error(codes.ResourceExhausted, "rate limit exceeded"), - expected: "rate_limited", - }, - { - name: "StatusCode_NotFound reqError maps to not_found", - err: newReqError(envoy_type.StatusCode_NotFound, "missing"), - expected: "not_found", - }, - { - name: "StatusCode_ServiceUnavailable reqError maps to no_capacity", - err: newReqError(envoy_type.StatusCode_ServiceUnavailable, "no free workers"), - expected: "no_capacity", - }, - { - name: "StatusCode_TooManyRequests reqError maps to rate_limited", - err: newReqError(envoy_type.StatusCode_TooManyRequests, "rate limited"), - expected: "rate_limited", - }, - { - name: "Unknown error maps to resume_error", - err: errors.New("internal storage glitch"), - expected: "resume_error", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := classifyOutcome(tc.err); got != tc.expected { - t.Errorf("classifyOutcome(%v) = %q, want %q", tc.err, got, tc.expected) - } - }) - } -} - -func TestRecordRouteDuration_Attributes(t *testing.T) { - reader := sdkmetric.NewManualReader() - mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - h, err := mp.Meter("atenet-router").Float64Histogram(routeDurationMetricName) - if err != nil { - t.Fatalf("failed to create histogram: %v", err) - } - - s := NewExtProcServer(50051, nil, h, ParkedRequestConfig{}, nil, false) - s.recordRouteDuration(context.Background(), 10*time.Millisecond, "team-a-ns", "tmpl-a", classifyOutcome(nil), string(ResumeOutcomeTriggered)) - - var rm metricdata.ResourceMetrics - if err := reader.Collect(context.Background(), &rm); err != nil { - t.Fatalf("Collect failed: %v", err) - } - - dp := rm.ScopeMetrics[0].Metrics[0].Data.(metricdata.Histogram[float64]).DataPoints[0] - wantAttrs := map[string]string{ - "ate.template.namespace": "team-a-ns", - "ate.template.name": "tmpl-a", - "ate.router.outcome": "ok", - "ate.router.resume": "triggered", - } - - for k, want := range wantAttrs { - val, exists := dp.Attributes.Value(attribute.Key(k)) - if !exists { - t.Errorf("missing metric attribute %q", k) - } else if val.AsString() != want { - t.Errorf("attribute %q = %q, want %q", k, val.AsString(), want) - } - } -} - func TestAddRoutingMutationsViaAuthority(t *testing.T) { mutation := &extprocv3.HeaderMutation{} addRoutingMutations("10.0.0.52:443", "actor-1.team-a.actors.resources.substrate.ate.dev", true, mutation) @@ -439,7 +319,7 @@ func TestAddRoutingMutationsViaAuthority(t *testing.T) { if got[strings.ToLower(atunnel.OriginalHostHeader)] != "actor-1.team-a.actors.resources.substrate.ate.dev" { t.Errorf("%s = %q", atunnel.OriginalHostHeader, got[strings.ToLower(atunnel.OriginalHostHeader)]) } - if got[authorityHeader] != "10.0.0.52:443" { - t.Errorf("%s = %q", authorityHeader, got[authorityHeader]) + if got[extproc.AuthorityHeader] != "10.0.0.52:443" { + t.Errorf("%s = %q", extproc.AuthorityHeader, got[extproc.AuthorityHeader]) } } diff --git a/cmd/atenet/internal/router/metrics.go b/cmd/atenet/internal/router/ingress/metrics.go similarity index 58% rename from cmd/atenet/internal/router/metrics.go rename to cmd/atenet/internal/router/ingress/metrics.go index 448ab3204d..29bc26e702 100644 --- a/cmd/atenet/internal/router/metrics.go +++ b/cmd/atenet/internal/router/ingress/metrics.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -22,58 +22,33 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" ) +// Request-parking instruments. parking.active is the live count of parked +// requests; parking.wait.duration is how long each request stayed parked +// (labeled by outcome); parking.rejected counts requests shed because the +// parking lot was full. const ( - routerServiceName = "atenet-router" - - // atenet.router.route.duration measures the latency from when the ext_proc handler receives a request - // (Envoy -> EPP) until the target worker endpoint is resolved - routeDurationMetricName = "atenet.router.route.duration" - - // Request-parking instruments. parking.active is the live count of parked - // requests; parking.wait.duration is how long each request stayed parked - // (labeled by outcome); parking.rejected counts requests shed because the - // parking lot was full. parkingActiveMetricName = "atenet.router.parking.active" parkingWaitMetricName = "atenet.router.parking.wait.duration" parkingRejectedMetricName = "atenet.router.parking.rejected" ) -// newRouteDurationHistogram creates the atenet.router.route.duration histogram from -// the global MeterProvider. -func newRouteDurationHistogram() (metric.Float64Histogram, error) { - h, err := otel.Meter(routerServiceName).Float64Histogram( - routeDurationMetricName, - metric.WithUnit("s"), - metric.WithDescription( - "latency between Substrate router receiving a request and resolving "+ - "the target worker endpoint, excluding actor execution and response", - ), - metric.WithExplicitBucketBoundaries( - 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, - 0.075, 0.1, 0.15, 0.2, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30, - ), - ) - if err != nil { - return nil, fmt.Errorf("create %s histogram: %w", routeDurationMetricName, err) - } - return h, nil -} - -// parkingMetrics bundles the OpenTelemetry instruments used by the parking lot. -// A nil *parkingMetrics is safe to use: every method becomes a no-op, which +// ParkingMetrics bundles the OpenTelemetry instruments used by the parking lot. +// A nil *ParkingMetrics is safe to use: every method becomes a no-op, which // keeps tests and metric-free deployments simple. -type parkingMetrics struct { +type ParkingMetrics struct { active metric.Int64UpDownCounter wait metric.Float64Histogram rejected metric.Int64Counter } -// newParkingMetrics creates the request-parking instruments from the global +// NewParkingMetrics creates the request-parking instruments from the global // MeterProvider. -func newParkingMetrics() (*parkingMetrics, error) { - meter := otel.Meter(routerServiceName) +func NewParkingMetrics() (*ParkingMetrics, error) { + meter := otel.Meter(extproc.ServiceName) active, err := meter.Int64UpDownCounter( parkingActiveMetricName, @@ -105,24 +80,24 @@ func newParkingMetrics() (*parkingMetrics, error) { return nil, fmt.Errorf("create %s counter: %w", parkingRejectedMetricName, err) } - return &parkingMetrics{active: active, wait: wait, rejected: rejected}, nil + return &ParkingMetrics{active: active, wait: wait, rejected: rejected}, nil } -func (m *parkingMetrics) addActive(ctx context.Context, delta int64) { +func (m *ParkingMetrics) addActive(ctx context.Context, delta int64) { if m == nil || m.active == nil { return } m.active.Add(ctx, delta) } -func (m *parkingMetrics) recordWait(ctx context.Context, d time.Duration, outcome parkOutcome) { +func (m *ParkingMetrics) recordWait(ctx context.Context, d time.Duration, outcome parkOutcome) { if m == nil || m.wait == nil { return } m.wait.Record(ctx, d.Seconds(), metric.WithAttributes(attribute.String("outcome", string(outcome)))) } -func (m *parkingMetrics) recordRejected(ctx context.Context) { +func (m *ParkingMetrics) recordRejected(ctx context.Context) { if m == nil || m.rejected == nil { return } diff --git a/cmd/atenet/internal/router/parking.go b/cmd/atenet/internal/router/ingress/parking.go similarity index 81% rename from cmd/atenet/internal/router/parking.go rename to cmd/atenet/internal/router/ingress/parking.go index fa0265366d..a00f66e9ad 100644 --- a/cmd/atenet/internal/router/parking.go +++ b/cmd/atenet/internal/router/ingress/parking.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -26,23 +26,31 @@ import ( // Default request-parking parameters. See ParkedRequestConfig for the meaning of each // field; these are also the flag defaults wired up in NewRouterCmd. const ( - defaultParkedRequestBudget = 5 * time.Second + DefaultParkedRequestBudget = 5 * time.Second - // defaultParkedRequestMax is sized together with the ext_proc cluster's + // DefaultParkedRequestMax is sized together with the ext_proc cluster's // circuit breaker (--extproc-max-requests, derived as twice the lot by default): each parked // request holds one ext_proc stream, i.e. one active request against that // cluster, for its entire wait. Startup validation keeps an explicit breaker >= the // lot, and the default pair (1024 lot / 2048 breaker) leaves equal headroom // for the fast path. See buildCluster in xds.go. - defaultParkedRequestMax = 1024 + DefaultParkedRequestMax = 1024 // Retry cadence between resume attempts while a request is parked: a gentle // exponential backoff. - defaultParkedRequestRetryInterval = 100 * time.Millisecond - defaultParkedRequestRetryFactor = 1.1 - defaultParkedRequestRetryJitter = 0.1 + DefaultParkedRequestRetryInterval = 100 * time.Millisecond + DefaultParkedRequestRetryFactor = 1.1 + DefaultParkedRequestRetryJitter = 0.1 ) +// ParkingStatus is a snapshot of the request-parking lot for the status page. +type ParkingStatus struct { + Enabled bool `json:"enabled"` + Active int `json:"active"` + MaxParked int `json:"max_parked"` + MaxWait string `json:"max_wait"` +} + // parkOutcome is the terminal disposition of a parked request. It is recorded // as the `outcome` label on the parking.wait.duration histogram. type parkOutcome string @@ -81,31 +89,31 @@ type ParkedRequestConfig struct { RetryJitter float64 } -// enabled reports whether request parking is active. Parking has no separate +// Enabled reports whether request parking is active. Parking has no separate // on/off switch: setting Max to 0 disables it, applying a fail-fast behavior // (no admission cap, no retry on pool saturation). -func (c ParkedRequestConfig) enabled() bool { return c.Max > 0 } +func (c ParkedRequestConfig) Enabled() bool { return c.Max > 0 } -// normalized returns the config with non-positive budget and retry parameters +// Normalized returns the config with non-positive budget and retry parameters // replaced by their defaults, so every consumer (the resumer's retry loop and // the Envoy ext_proc timeout) sees the same effective values. -func (c ParkedRequestConfig) normalized() ParkedRequestConfig { +func (c ParkedRequestConfig) Normalized() ParkedRequestConfig { if c.Budget <= 0 { - c.Budget = defaultParkedRequestBudget + c.Budget = DefaultParkedRequestBudget } if c.RetryInterval <= 0 { - c.RetryInterval = defaultParkedRequestRetryInterval + c.RetryInterval = DefaultParkedRequestRetryInterval } if c.RetryFactor == 0 { - c.RetryFactor = defaultParkedRequestRetryFactor + c.RetryFactor = DefaultParkedRequestRetryFactor } return c } -// validate rejects retry parameters that would make parking misbehave rather +// Validate rejects retry parameters that would make parking misbehave rather // than merely differ: a factor below 1 shrinks delays toward zero and turns // the parked retry loop into a hot loop against the control plane. -func (c ParkedRequestConfig) validate() error { +func (c ParkedRequestConfig) Validate() error { if c.RetryFactor != 0 && c.RetryFactor < 1.0 { return fmt.Errorf("parked-request retry factor must be >= 1.0, got %v", c.RetryFactor) } @@ -115,15 +123,15 @@ func (c ParkedRequestConfig) validate() error { return nil } -// defaultParkedRequestConfig returns the built-in parking configuration +// DefaultParkedRequestConfig returns the built-in parking configuration // (matching the NewRouterCmd flag defaults). -func defaultParkedRequestConfig() ParkedRequestConfig { +func DefaultParkedRequestConfig() ParkedRequestConfig { return ParkedRequestConfig{ - Budget: defaultParkedRequestBudget, - Max: defaultParkedRequestMax, - RetryInterval: defaultParkedRequestRetryInterval, - RetryFactor: defaultParkedRequestRetryFactor, - RetryJitter: defaultParkedRequestRetryJitter, + Budget: DefaultParkedRequestBudget, + Max: DefaultParkedRequestMax, + RetryInterval: DefaultParkedRequestRetryInterval, + RetryFactor: DefaultParkedRequestRetryFactor, + RetryJitter: DefaultParkedRequestRetryJitter, } } @@ -136,13 +144,13 @@ func defaultParkedRequestConfig() ParkedRequestConfig { // accounting, applying the router's fail-fast behavior. type parkingLot struct { cfg ParkedRequestConfig - metrics *parkingMetrics + metrics *ParkingMetrics mu sync.Mutex active int // current number of occupied slots; guarded by mu } -func newParkingLot(cfg ParkedRequestConfig, m *parkingMetrics) *parkingLot { +func newParkingLot(cfg ParkedRequestConfig, m *ParkingMetrics) *parkingLot { return &parkingLot{cfg: cfg, metrics: m} } @@ -153,7 +161,7 @@ func newParkingLot(cfg ParkedRequestConfig, m *parkingMetrics) *parkingLot { // waiting. When parking is disabled every request is admitted and no slot // accounting or metrics are recorded. func (l *parkingLot) enter(ctx context.Context) (release func(outcome parkOutcome), ok bool) { - if !l.cfg.enabled() { + if !l.cfg.Enabled() { return func(parkOutcome) {}, true } @@ -198,7 +206,7 @@ func (l *parkingLot) activeCount() int { // status returns a snapshot of the lot for the /statusz page. func (l *parkingLot) status() ParkingStatus { return ParkingStatus{ - Enabled: l.cfg.enabled(), + Enabled: l.cfg.Enabled(), Active: l.activeCount(), MaxParked: l.cfg.Max, MaxWait: l.cfg.Budget.String(), diff --git a/cmd/atenet/internal/router/parking_test.go b/cmd/atenet/internal/router/ingress/parking_test.go similarity index 99% rename from cmd/atenet/internal/router/parking_test.go rename to cmd/atenet/internal/router/ingress/parking_test.go index 62f20f18da..ad8a529e68 100644 --- a/cmd/atenet/internal/router/parking_test.go +++ b/cmd/atenet/internal/router/ingress/parking_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" diff --git a/cmd/atenet/internal/router/resumer.go b/cmd/atenet/internal/router/ingress/resumer.go similarity index 96% rename from cmd/atenet/internal/router/resumer.go rename to cmd/atenet/internal/router/ingress/resumer.go index e6b01f7059..ce4bd943ff 100644 --- a/cmd/atenet/internal/router/resumer.go +++ b/cmd/atenet/internal/router/ingress/resumer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -20,6 +20,7 @@ import ( "sync/atomic" "time" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -114,9 +115,9 @@ type resumerOption func(*ActorResumer) // resume is retried, at cfg's retry cadence, for up to cfg's budget. When // disabled, the resumer applies fail-fast-on-capacity behavior. func withParking(cfg ParkedRequestConfig) resumerOption { - cfg = cfg.normalized() + cfg = cfg.Normalized() return func(r *ActorResumer) { - r.parkEnabled = cfg.enabled() + r.parkEnabled = cfg.Enabled() if r.parkEnabled { r.budget = cfg.Budget } @@ -128,8 +129,8 @@ func NewActorResumer(apiClient ateapipb.ControlClient, opts ...resumerOption) *A r := &ActorResumer{ apiClient: apiClient, budget: failFastResumeBudget, - backoff: resumeBackoff(defaultParkedRequestRetryInterval, - defaultParkedRequestRetryFactor, defaultParkedRequestRetryJitter), + backoff: resumeBackoff(DefaultParkedRequestRetryInterval, + DefaultParkedRequestRetryFactor, DefaultParkedRequestRetryJitter), } for _, opt := range opts { opt(r) @@ -161,7 +162,7 @@ func (r *ActorResumer) retryable(err error) bool { // requests within the process and, when parking is enabled, holds the request // while retrying transient failures until the budget elapses. func (r *ActorResumer) ResumeActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, ResumeOutcome, error) { - ctx, span := otel.Tracer(routerServiceName).Start(ctx, "ResumeActor", + ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ResumeActor", trace.WithAttributes(ateattr.ActorRefAttributes(actorRef)...)) defer span.End() diff --git a/cmd/atenet/internal/router/resumer_test.go b/cmd/atenet/internal/router/ingress/resumer_test.go similarity index 99% rename from cmd/atenet/internal/router/resumer_test.go rename to cmd/atenet/internal/router/ingress/resumer_test.go index b632789a1a..225e592577 100644 --- a/cmd/atenet/internal/router/resumer_test.go +++ b/cmd/atenet/internal/router/ingress/resumer_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -527,7 +527,7 @@ func TestResumeBackoffHasNoCap(t *testing.T) { // Steps the moment the delay reaches Cap, which would end parking retries far // short of the budget (a 2s Cap stops the loop in ~7 steps / ~5s). The budget // context — not the step count or a cap — must bound how long a request parks. - b := resumeBackoff(defaultParkedRequestRetryInterval, defaultParkedRequestRetryFactor, defaultParkedRequestRetryJitter) + b := resumeBackoff(DefaultParkedRequestRetryInterval, DefaultParkedRequestRetryFactor, DefaultParkedRequestRetryJitter) if b.Cap != 0 { t.Errorf("resume backoff must not set Cap (it would stop retries at the cap); got %v", b.Cap) } diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 49fd3e5036..5ce654b2ab 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -16,11 +16,13 @@ package router import ( "context" + "crypto/x509" "errors" "fmt" "log/slog" "net" "net/http" + "os" "os/signal" "syscall" @@ -37,6 +39,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/serverboot" v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -60,51 +65,60 @@ func init() { type RouterServer struct { cfg routerConfig - Cmd *cobra.Command - k8sClient client.Client - clientset kubernetes.Interface - apiClient ateapipb.ControlClient - extprocSrv *ExtProcServer - health *routerHealth - atStore atStore + Cmd *cobra.Command + k8sClient client.Client + clientset kubernetes.Interface + apiClient ateapipb.ControlClient + // extprocSrv is the ext_proc mux. Which handlers it carries — ingress, + // egress, or both — follows cfg.Mode. + extprocSrv *extproc.Server + // ingressHandler is the ingress handler registered on extprocSrv, kept for + // the status page's parking snapshot. Nil in egress-only mode. + ingressHandler *ingress.Handler + health *routerHealth + atStore atStore } func NewRouterServer(cfg routerConfig) (*RouterServer, error) { var k8sClient client.Client var clientset kubernetes.Interface + var store atStore - if cfg.TemplatesFile == "" { - k8sCfg, err := config.GetConfig() - if err != nil { - if cfg.Kubeconfig != "" { - k8sCfg, err = clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) - if err != nil { - return nil, fmt.Errorf("failed to read config from path %s: %w", cfg.Kubeconfig, err) + // Only ingress needs Kubernetes: it is the ActorTemplate controller and the + // xDS server that read from it. An egress-only instance is pure ext_proc and + // deliberately runs without any cluster access at all, so do not even build + // the clients — in-cluster config would only fail for want of RBAC. + if cfg.Mode.ServesIngress() { + if cfg.TemplatesFile == "" { + k8sCfg, err := config.GetConfig() + if err != nil { + if cfg.Kubeconfig != "" { + k8sCfg, err = clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to read config from path %s: %w", cfg.Kubeconfig, err) + } + } else { + return nil, fmt.Errorf("unable to establish Kubernetes configuration parameters: %w", err) } - } else { - return nil, fmt.Errorf("unable to establish Kubernetes configuration parameters: %w", err) } - } - slog.Info("Connecting to Kubernetes API server", slog.String("host", k8sCfg.Host)) + slog.Info("Connecting to Kubernetes API server", slog.String("host", k8sCfg.Host)) - k8sClient, err = client.New(k8sCfg, client.Options{ - Scheme: scheme, - }) - if err != nil { - return nil, fmt.Errorf("failed to initialize cluster client: %w", err) - } + k8sClient, err = client.New(k8sCfg, client.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize cluster client: %w", err) + } - clientset, err = kubernetes.NewForConfig(k8sCfg) - if err != nil { - return nil, fmt.Errorf("failed to initialize core client: %w", err) - } - } + clientset, err = kubernetes.NewForConfig(k8sCfg) + if err != nil { + return nil, fmt.Errorf("failed to initialize core client: %w", err) + } - var store atStore - if cfg.TemplatesFile != "" { - store = newFileATStore(cfg.TemplatesFile) - } else { - store = newk8sATStore(k8sClient) + store = newk8sATStore(k8sClient) + } else { + store = newFileATStore(cfg.TemplatesFile) + } } return &RouterServer{ @@ -119,7 +133,7 @@ func (s *RouterServer) Run(ctx context.Context) error { // shutdownCtx signals SIGTERM/SIGINT; kept separate from the work context // so in-flight ext_proc streams (parked requests, most of all) are not // cancelled the moment the signal arrives. drainOnShutdown drives the - // shutdown sequence: readiness flip → route-drain delay → Envoy drain → + // shutdown sequence: readiness flip → route-drain delay → dataplane drain → // ext_proc drain → stop the rest. shutdownCtx, stopSignals := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stopSignals() @@ -136,11 +150,11 @@ func (s *RouterServer) Run(ctx context.Context) error { if err := s.cfg.validate(); err != nil { return fmt.Errorf("invalid router configuration: %w", err) } - parkCfg := s.cfg.ParkedRequest.normalized() + parkCfg := s.cfg.ParkedRequest.Normalized() - // The drain-complete marker persists container restarts (emptyDir); a - // stale one would release the Envoy preStop hook the moment a later drain - // begins. + // The drain-complete marker persists container restarts (emptyDir); a stale + // one would release the dataplane container's preStop hook the moment a + // later drain begins. removeStaleDrainMarker(ctx, s.cfg.DrainCompleteFile) serverboot.InitLogger() @@ -154,7 +168,7 @@ func (s *RouterServer) Run(ctx context.Context) error { // sampler and Envoy's RandomSampling percent cannot drift. sampling := serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(dataPlaneTraceRatio)) tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ - ServiceName: routerServiceName, + ServiceName: extproc.ServiceName, Sampling: sampling, }) if err != nil { @@ -162,7 +176,7 @@ func (s *RouterServer) Run(ctx context.Context) error { } defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown) - mp, err := serverboot.InitMetrics(ctx, routerServiceName) + mp, err := serverboot.InitMetrics(ctx, extproc.ServiceName) if err != nil { return fmt.Errorf("failed to initialize metrics: %w", err) } @@ -201,26 +215,62 @@ func (s *RouterServer) Run(ctx context.Context) error { s.apiClient = ateapipb.NewControlClient(conn) slog.InfoContext(ctx, "Starting substrate router subsystem", + slog.String("mode", string(s.cfg.Mode)), slog.Bool("standalone", s.cfg.Standalone), slog.String("atenet_router", string(s.cfg.atenetRouter()))) g, ctx := errgroup.WithContext(ctx) - if s.extprocSrv == nil { - routeDuration, err := newRouteDurationHistogram() + // Register one handler per direction this instance serves. The mux refuses + // any direction missing from this map, so the mode is enforced here rather + // than merely advertised. + handlers := extproc.Handlers{} + if s.cfg.Mode.ServesIngress() { + parkMetrics, err := ingress.NewParkingMetrics() if err != nil { - return fmt.Errorf("failed to create route-duration histogram: %w", err) + return fmt.Errorf("failed to create parking metrics: %w", err) } - parkMetrics, err := newParkingMetrics() + s.ingressHandler = ingress.New(s.apiClient, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority()) + handlers[s.ingressHandler.Direction()] = s.ingressHandler + } + if s.cfg.Mode.ServesEgress() { + // Load the actor-identity CA up front so a missing or unusable bundle + // fails startup, rather than turning into a 503 on the first actor + // egress attempt. An unset flag leaves the handler with no roots, which + // denies every CONNECT — see egress.New. + var actorIdentityRoots *x509.CertPool + if s.cfg.ActorIdentityCAFile != "" { + pemBytes, err := os.ReadFile(s.cfg.ActorIdentityCAFile) + if err != nil { + return fmt.Errorf("reading --actor-identity-ca-file: %w", err) + } + actorIdentityRoots, err = egress.LoadActorIdentityRoots(pemBytes) + if err != nil { + return fmt.Errorf("loading --actor-identity-ca-file %q: %w", s.cfg.ActorIdentityCAFile, err) + } + } + egressHandler := egress.New(s.apiClient, actorIdentityRoots) + handlers[egressHandler.Direction()] = egressHandler + } + + if s.extprocSrv == nil { + routeDuration, err := extproc.NewRouteDurationHistogram() if err != nil { - return fmt.Errorf("failed to create parking metrics: %w", err) + return fmt.Errorf("failed to create route-duration histogram: %w", err) } - s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority()) + s.extprocSrv = extproc.NewServer(s.cfg.ExtprocPort, routeDuration, handlers) } + s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg) - if err := s.startDataplane(ctx, g, parkCfg, sampling.RootSamplingPercent()); err != nil { - return err + // The ingress control plane — the xDS server and the ActorTemplate + // controller — configures the *ingress* dataplane. The egress gateway is + // statically configured, so an egress-only instance runs neither and needs + // no Kubernetes access. + if s.cfg.Mode.ServesIngress() { + if err := s.startDataplane(ctx, g, parkCfg, sampling.RootSamplingPercent()); err != nil { + return err + } } // Start periodic service checking logic @@ -231,7 +281,7 @@ func (s *RouterServer) Run(ctx context.Context) error { }) // Start ExtProc Server. Driven by the drain sequence rather than context - // cancel: ext_proc is failClosed, so it must outlive Envoy's drain. + // cancel: ext_proc is failClosed, so it must outlive the dataplane's drain. extprocGRPC := s.extprocSrv.NewGRPCServer() g.Go(func() error { slog.InfoContext(ctx, "Starting ExtProc Server", slog.Int("port", s.cfg.ExtprocPort)) @@ -306,8 +356,8 @@ func (s *RouterServer) Run(ctx context.Context) error { // OTEL_EXPORTER_OTLP_ENDPOINT, which the router's own exporter reads too and // which legitimately carries forms Envoy's plaintext tracer cluster cannot // reach — an https collector, most of all. Refusing to start would take the -// xDS control plane for every Envoy in the mesh down over a tracing endpoint -// that works fine for its other reader. Losing Envoy's spans is the smaller +// xDS control plane for every ingress Envoy down over a tracing endpoint that +// works fine for its other reader. Losing Envoy's spans is the smaller // failure, so take it and say so loudly. func setOtlpCollector(ctx context.Context, xdsSrv *XdsServer, addr string) { if err := xdsSrv.SetOtlpCollector(addr); err != nil { diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index 6a76253730..6c950a0ccb 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -25,137 +25,38 @@ import ( "os" "runtime/debug" "strings" - "sync" "time" "github.com/spf13/pflag" "go.opentelemetry.io/otel" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) var BuildTag = "dev" -type RecordedQuery struct { - Timestamp time.Time `json:"timestamp"` - Client string `json:"client"` - Host string `json:"host"` - Path string `json:"path"` - Method string `json:"method"` - Action string `json:"action"` - Target string `json:"target"` - Duration time.Duration `json:"duration"` -} - -type QueryRecorder struct { - mu sync.RWMutex - queries []RecordedQuery - size int - index int -} - -func NewQueryRecorder(size int) *QueryRecorder { - return &QueryRecorder{ - queries: make([]RecordedQuery, 0, size), - size: size, - } -} - -func (qr *QueryRecorder) Add(q RecordedQuery) { - if qr == nil { - return - } - - qr.mu.Lock() - defer qr.mu.Unlock() - - if len(qr.queries) < qr.size { - qr.queries = append(qr.queries, q) - } else { - qr.queries[qr.index] = q - qr.index = (qr.index + 1) % qr.size - } -} - -func (qr *QueryRecorder) Get() []RecordedQuery { - if qr == nil { - return nil - } - - qr.mu.RLock() - defer qr.mu.RUnlock() - - n := len(qr.queries) - if n == 0 { - return nil - } - - res := make([]RecordedQuery, n) - if n < qr.size { - for i := 0; i < n; i++ { - res[i] = qr.queries[n-1-i] - } - } else { - for i := 0; i < n; i++ { - pos := (qr.index - 1 - i + n) % n - res[i] = qr.queries[pos] - } - } - - return res -} - -// redactPath drops the query string, which may carry credentials (CWE-598). -func redactPath(path string) string { - p, _, _ := strings.Cut(path, "?") - return p -} - -func (qr *QueryRecorder) AddRouterRequest( - start time.Time, - duration time.Duration, - action, - target string, - m *requestMetadata, -) { - qr.Add(RecordedQuery{ - Timestamp: start, - Client: m.headers[authorityHeader], - Host: m.host, - Path: redactPath(m.path), - Method: m.headers[":method"], - Action: action, - Target: target, - Duration: duration, - }) -} - type TemplateInfo struct { Name string `json:"name"` Namespace string `json:"namespace"` } -// ParkingStatus is a snapshot of the request-parking lot for the status page. -type ParkingStatus struct { - Enabled bool `json:"enabled"` - Active int `json:"active"` - MaxParked int `json:"max_parked"` - MaxWait string `json:"max_wait"` -} - type DashboardContext struct { - BuildTag string `json:"build_tag"` - RouterClusterIP string `json:"router_cluster_ip"` - Namespace string `json:"namespace"` - HttpPort int `json:"port_http"` - XdsPort int `json:"port_xds"` - ExtprocPort int `json:"port_extproc"` - StatusPort int `json:"status_port"` - Args string `json:"args"` - Flags map[string]string `json:"flags"` - Queries []FormattedQuery `json:"queries"` - Health RouterHealthReport `json:"health"` - Templates []TemplateInfo `json:"templates"` - Parking ParkingStatus `json:"parking"` + BuildTag string `json:"build_tag"` + RouterClusterIP string `json:"router_cluster_ip"` + Namespace string `json:"namespace"` + Mode Mode `json:"mode"` + HttpPort int `json:"port_http"` + XdsPort int `json:"port_xds"` + ExtprocPort int `json:"port_extproc"` + StatusPort int `json:"status_port"` + Args string `json:"args"` + Flags map[string]string `json:"flags"` + Queries []FormattedQuery `json:"queries"` + Health RouterHealthReport `json:"health"` + Templates []TemplateInfo `json:"templates"` + Parking ingress.ParkingStatus `json:"parking"` } type FormattedQuery struct { @@ -187,7 +88,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string { } func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { - ctx, span := otel.Tracer(routerServiceName).Start(req.Context(), "handleStatusz") + ctx, span := otel.Tracer(extproc.ServiceName).Start(req.Context(), "handleStatusz") defer span.End() ctx, cancel := context.WithTimeout(ctx, 3*time.Second) @@ -213,9 +114,9 @@ func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { }) } - var rawQueries []RecordedQuery - if s.extprocSrv != nil && s.extprocSrv.recorder != nil { - rawQueries = s.extprocSrv.recorder.Get() + var rawQueries []extproc.RecordedQuery + if s.extprocSrv != nil { + rawQueries = s.extprocSrv.Queries() } formattedQueries := make([]FormattedQuery, len(rawQueries)) @@ -253,15 +154,17 @@ func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { } } - var parking ParkingStatus - if s.extprocSrv != nil { - parking = s.extprocSrv.parking.status() + // Parking belongs to the ingress handler; an egress-only instance has none. + var parking ingress.ParkingStatus + if s.ingressHandler != nil { + parking = s.ingressHandler.ParkingStatus() } data := DashboardContext{ BuildTag: buildInfo, RouterClusterIP: routerIP, Namespace: s.cfg.Namespace, + Mode: s.cfg.Mode, HttpPort: s.cfg.HttpPort, XdsPort: s.cfg.XdsPort, ExtprocPort: s.cfg.ExtprocPort, diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index 2f6e97cbd0..5be288f97a 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -33,6 +33,9 @@ import ( "strings" "testing" "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func TestStatuszEndpoint(t *testing.T) { @@ -68,6 +71,7 @@ func TestStatuszEndpoint(t *testing.T) { ExtProcMaxRequests: defaultExtProcMaxRequests, TemplatesFile: tmpFile.Name(), MetricsAddr: "127.0.0.1:0", + ParkedRequest: ingress.DefaultParkedRequestConfig(), Auth: authConfig{ AteapiCAFile: caPath, AteapiClientCertPath: clientCertPath, @@ -79,13 +83,13 @@ func TestStatuszEndpoint(t *testing.T) { t.Fatalf("Failed generating router server: %v", err) } - srv.extprocSrv = NewExtProcServer(cfg.ExtprocPort, &mockClient{}, nil, defaultParkedRequestConfig(), nil, false) + srv.extprocSrv = extproc.NewServer(cfg.ExtprocPort, nil, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Inject recorded queries - srv.extprocSrv.recorder.Add(RecordedQuery{ + srv.extprocSrv.Recorder().Add(extproc.RecordedQuery{ Timestamp: time.Now(), Client: "127.0.0.1", Host: "example.com", @@ -164,8 +168,8 @@ func TestStatuszEndpoint(t *testing.T) { if !dashboard.Parking.Enabled { t.Errorf("expected parking reported as enabled in status JSON") } - if dashboard.Parking.MaxParked != defaultParkedRequestMax { - t.Errorf("expected parking max_parked %d, got %d", defaultParkedRequestMax, dashboard.Parking.MaxParked) + if dashboard.Parking.MaxParked != ingress.DefaultParkedRequestMax { + t.Errorf("expected parking max_parked %d, got %d", ingress.DefaultParkedRequestMax, dashboard.Parking.MaxParked) } } diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index bcf07c7e63..d2ec58dec1 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -58,6 +58,8 @@ import ( cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" serverv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) const ( @@ -76,12 +78,10 @@ const ( httpProtocolOptionsName = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" // OriginalDstClusterName routes actor traffic to the worker's atunnel - // ingress by the IP:port the ext_proc puts in OriginalDstHeader, while the - // request :authority stays the actor DNS name so atunnel can identify the - // active actor. + // ingress by the IP:port the ext_proc puts in ingress.OriginalDstHeader, + // while the request :authority stays the actor DNS name so atunnel can + // identify the active actor. OriginalDstClusterName = "actor_original_dst" - // OriginalDstHeader carries the resolved worker atunnel address (IP:443). - OriginalDstHeader = "x-ate-original-dst" ) // defaultExtProcMessageTimeout is Envoy's per-message ext_proc response timeout @@ -90,7 +90,7 @@ const ( const defaultExtProcMessageTimeout = 5 * time.Second // defaultExtProcMaxRequests is the circuit-breaker max_requests set on the -// ext_proc cluster: defaultParkedRequestMax plus equal fast-path headroom, so a +// ext_proc cluster: ingress.DefaultParkedRequestMax plus equal fast-path headroom, so a // full parking lot cannot starve the millisecond-scale header exchanges of // requests to already-running actors. See buildCluster. const defaultExtProcMaxRequests = 2048 @@ -621,7 +621,7 @@ func (x *XdsServer) buildUpstreamTransportSocket() *corev3.TransportSocket { } // buildOriginalDstCluster dials the exact worker atunnel address supplied by -// the ext_proc in OriginalDstHeader. Unlike the dynamic_forward_proxy cluster, +// the ext_proc in ingress.OriginalDstHeader. Unlike the dynamic_forward_proxy cluster, // it does not derive the destination from :authority, so the request keeps the // actor DNS name as its Host for atunnel to authorize. mTLS to atunnel is // applied via the shared upstream transport socket (SPIFFE URI validation). @@ -636,7 +636,7 @@ func (x *XdsServer) buildOriginalDstCluster() *clusterv3.Cluster { LbConfig: &clusterv3.Cluster_OriginalDstLbConfig_{ OriginalDstLbConfig: &clusterv3.Cluster_OriginalDstLbConfig{ UseHttpHeader: true, - HttpHeaderName: OriginalDstHeader, + HttpHeaderName: ingress.OriginalDstHeader, }, }, } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 3b4e6aef0b..98287eae82 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -43,6 +43,8 @@ import ( secretgrpc "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func TestXdsServer_UpdateSnapshot(t *testing.T) { @@ -540,8 +542,8 @@ func TestXdsServer_ExtProcCircuitBreaker(t *testing.T) { if got != uint32(defaultExtProcMaxRequests) { t.Errorf("default max_requests = %d, want %d", got, defaultExtProcMaxRequests) } - if got < uint32(defaultParkedRequestMax) { - t.Errorf("default breaker (%d) below the default lot (%d): a full lot would be truncated by Envoy", got, defaultParkedRequestMax) + if got < uint32(ingress.DefaultParkedRequestMax) { + t.Errorf("default breaker (%d) below the default lot (%d): a full lot would be truncated by Envoy", got, ingress.DefaultParkedRequestMax) } }) diff --git a/demos/egress/README.md b/demos/egress/README.md new file mode 100644 index 0000000000..6e5dcf85ad --- /dev/null +++ b/demos/egress/README.md @@ -0,0 +1,150 @@ +# Egress Demo — Pluggable Egress Networking + +This demo shows an Actor's outbound traffic being **transparently tunneled through an +egress gateway** and **authenticated by actor identity**, end to end. + +The Actor is a tiny service that accepts `{"url":"..."}`, performs an HTTP `GET`, and returns +the upstream response. The Actor believes it is dialing plain HTTP directly — but its egress is +intercepted and carried over mTLS to a gateway that verifies who is making the request. + +## What it demonstrates + +``` + ┌──────────────── ateom worker pod ─────────────────┐ + │ Actor (gVisor) │ + │ GET http://:80/ (plain HTTP) │ + │ │ │ + │ ▼ nftables REDIRECT │ + │ atunnel egress ──(mTLS with the actor's own │ + │ │ certificate + bare CONNECT) │ + └────────┼────────────────────────────────────────────┘ + ▼ + ┌──────────── atenet-egress pod ───────────────────┐ + │ Envoy egress gateway │ + │ • downstream mTLS, trusted_ca = actor-id CA │ + │ • terminates HTTP CONNECT │ + │ • ext_proc ──(localhost)──► atenet router (ext_proc sidecar) + │ (forwards the peer chain │ verify chain + ActorIdentity extension + │ as x-forwarded-client-cert)│ GetActor → UID must match, must be RUNNING + │ • dynamic_forward_proxy │ allow / deny 403 + │ │ + └───────────┼───────────────────────────────────────┘ + ▼ + real destination (the CONNECT authority, an IP:port) +``` + +1. **Guide 1 — gateway accepts CONNECT + mTLS.** `atenet-egress` is an Envoy dynamic-forward-proxy + that terminates the actor's mTLS `CONNECT` and tunnels to the requested destination. +2. **Guide 2 — transparent interception.** `nftables` REDIRECTs actor TCP egress into `atunnel`, + which wraps it in mTLS + `CONNECT`. +3. **Guide 3 — HTTP-only actors, identity carried by the certificate.** The Actor only dials plain + HTTP. atunnel presents the actor's own certificate — minted per actor by ateapi off the + actor-identity CA, carrying an `ActorIdentity` X.509 extension — and sends a bare `CONNECT` + with no identity headers at all. +4. **Identity authentication.** Envoy requires a client certificate signed by the actor-identity + CA, so a non-actor client is refused at the handshake. It then forwards the verified chain to + `ext_proc` as `x-forwarded-client-cert`, and the **atenet router** (co-located in the gateway + pod as an ext_proc sidecar, the same binary that serves ingress, started with `--mode=egress`) + re-verifies the chain, requires exactly one `ActorIdentity` extension with `purpose: atunnel`, + and calls the ate API (`GetActor`). It returns **403** unless the certified **UID** matches a + real, `RUNNING` actor. This mirrors the ingress gateway's dataplane + ext_proc co-location; a + standalone/shared ext_proc is a future step. + +## Components + +- **Egress app (`main.go`)** — the Actor: `POST /` with `{"url":"..."}` → fetches it → returns + status + body. +- **Egress gateway** — `manifests/ate-install/atenet-egress.yaml`. One pod, two containers: + an Envoy (`envoy`) and the atenet router ext_proc (`ext-proc`, `--mode=egress`), called over + localhost. In egress mode the router serves the egress ext_proc handler only — no xDS server, + no ActorTemplate controller, and no Kubernetes access at all. +- **Egress opt-in** — `ate-api-server --egress-gateway-address=atenet-egress.ate-system.svc:443` + (set in `manifests/ate-install/ate-api-server.yaml`). ateapi stamps the address onto every + atelet `Run`/`Restore`, which turns on tunneled egress cluster-wide. +- **Actor-identity trust** — the gateway mounts the `actor-id-ca-certs` Secret, a cert-only copy of + the actor-identity CA root that `hack/install-ate.sh` derives from `actor-id-ca-pool` (which also + holds the CA signing key and is deliberately *not* mounted here). + +## Prerequisites + +- A kind cluster with Agent Substrate installed (`hack/create-kind-cluster.sh` then + `hack/install-ate-kind.sh --deploy-ate-system`). Egress is enabled by the ateapi flag above. +- `ko`, `kubectl`, and `kubectl-ate` (`go install ./cmd/kubectl-ate`). + +## Deploy the demo fixture + +```bash +./hack/install-ate.sh --deploy-demo-egress +kubectl wait --for=condition=Ready actortemplate/egress -n ate-demo-egress --timeout=5m +``` + +## Run the automated test (easiest) + +```bash +./demos/egress/test-egress.sh +``` + +It deploys an in-cluster HTTP target, creates & resumes an Actor, then asserts: + +- **positive** — a real Actor's egress reaches the target (`HTTP 200`) *through the gateway* + (the target sees the gateway's IP as its client), and the gateway logs the CONNECT against the + actor's certificate SAN; +- **negative** — a pod holding a valid *pod* identity but no actor certificate cannot open a + tunnel at all: the gateway's `trusted_ca` is the actor-identity CA, so the mTLS handshake is + refused before any CONNECT is answered. + +Add `--cleanup` to remove everything the script created. + +## Manual walkthrough + +```bash +# 1. An in-cluster target the Actor will fetch (any HTTP server works). +kubectl create namespace egress-target +kubectl -n egress-target create deployment whoami --image=traefik/whoami +kubectl -n egress-target expose deployment whoami --port=80 +TARGET_IP=$(kubectl -n egress-target get svc whoami -o jsonpath='{.spec.clusterIP}') + +# 2. Create and resume an Actor. +kubectl ate create atespace demo +kubectl ate create actor egress-demo -a demo --template ate-demo-egress/egress +kubectl ate resume actor egress-demo -a demo # wait for STATUS_RUNNING + +# 3. Drive the Actor's egress through the ingress gateway. +kubectl -n ate-system port-forward service/atenet-router 8000:80 & +curl -s -X POST http://localhost:8000/ \ + -H 'Host: egress-demo.demo.actors.resources.substrate.ate.dev' \ + -H 'Content-Type: application/json' \ + -d "{\"url\":\"http://${TARGET_IP}:80/\"}" +``` + +### What to observe + +```bash +# The egress gateway logs each tunneled CONNECT against the verified peer certificate: +kubectl -n ate-system logs deploy/atenet-egress | grep '\[egress\]' +# [egress] authority=:80 peer_san=spiffe://substrate-actor.local/atespace/demo/actor/egress-demo … code=200 … + +# The co-located ext_proc sidecar logs the identity decision, including the UID it authorized on: +kubectl -n ate-system logs deploy/atenet-egress -c ext-proc | grep -i 'egress identity\|egress denied' +# egress identity authenticated atespace=demo actor=egress-demo actorUid=… destination=:80 +``` + +The `whoami` body shows `RemoteAddr: ` — proof the request egressed +*through* the gateway rather than directly. + +## Notes / limitations + +- This milestone **authenticates** identity (is this a real, running actor?). **Authorizing** + egress by destination and injecting upstream credentials/tokens is a follow-up, implemented in + the same `ext_proc` (policy API TBD). +- Identity comes entirely from the actor certificate: the atespace, actor name, and UID are read + out of the `ActorIdentity` extension and the UID is matched against the live actor, so a + certificate cannot survive its actor being deleted and recreated under the same name. Nothing + the actor can write into the CONNECT contributes to the decision. + +## Cleanup + +```bash +./demos/egress/test-egress.sh --cleanup +./hack/install-ate.sh --delete-demo-egress +``` diff --git a/demos/egress/egress.yaml.tmpl b/demos/egress/egress.yaml.tmpl new file mode 100644 index 0000000000..e6f9b080f2 --- /dev/null +++ b/demos/egress/egress.yaml.tmpl @@ -0,0 +1,56 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-demo-egress + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: egress + namespace: ate-demo-egress + labels: + workload: egress +spec: + replicas: 2 + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: egress + namespace: ate-demo-egress +spec: + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + containers: + - name: egress + image: ko://github.com/agent-substrate/substrate/demos/egress + command: ["/ko-app/egress"] + readyz: + httpGet: + path: /readyz + port: 80 + workerSelector: + matchLabels: + workload: egress + snapshotsConfig: + onPause: Full + onCommit: Full + location: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/egress/main.go b/demos/egress/main.go new file mode 100644 index 0000000000..4c0288e12e --- /dev/null +++ b/demos/egress/main.go @@ -0,0 +1,125 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command egress is a small HTTP service for demonstrating per-Actor egress +// policy. It accepts a URL, fetches it, and returns the upstream response. +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "time" +) + +const ( + listenAddress = ":80" + maxRequestBody = 64 << 10 + maxResponseBody = 1 << 20 + requestTimeout = 15 * time.Second +) + +type fetchRequest struct { + URL string `json:"url"` +} + +type fetchResponse struct { + StatusCode int `json:"statusCode,omitempty"` + Body string `json:"body,omitempty"` + Error string `json:"error,omitempty"` +} + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + + client := &http.Client{Timeout: requestTimeout} + slog.Info("starting egress demo", "address", listenAddress) + if err := http.ListenAndServe(listenAddress, newHandler(client)); err != nil { + slog.Error("egress demo stopped", "error", err) + os.Exit(1) + } +} + +func newHandler(client *http.Client) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "ok\n") + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + writeJSON(w, http.StatusMethodNotAllowed, fetchResponse{Error: "method must be POST"}) + return + } + + var input fetchRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRequestBody)) + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("invalid JSON payload: %v", err)}) + return + } + if err := validateURL(input.URL); err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: err.Error()}) + return + } + + outbound, err := http.NewRequestWithContext(r.Context(), http.MethodGet, input.URL, nil) + if err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("invalid URL: %v", err)}) + return + } + if traceparent := r.Header.Get("traceparent"); traceparent != "" { + outbound.Header.Set("traceparent", traceparent) + } + response, err := client.Do(outbound) + if err != nil { + writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("request failed: %v", err)}) + return + } + defer response.Body.Close() + + body, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBody)) + if err != nil { + writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("reading response: %v", err)}) + return + } + writeJSON(w, response.StatusCode, fetchResponse{StatusCode: response.StatusCode, Body: string(body)}) + }) + return mux +} + +func validateURL(raw string) error { + parsed, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return fmt.Errorf("URL scheme must be http or https") + } + if parsed.Hostname() == "" { + return fmt.Errorf("URL must include a hostname") + } + return nil +} + +func writeJSON(w http.ResponseWriter, status int, response fetchResponse) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(response) +} diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go new file mode 100644 index 0000000000..6cd203732f --- /dev/null +++ b/demos/egress/main_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestFetch(t *testing.T) { + const traceparent = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodGet { + t.Errorf("upstream method = %s, want GET", r.Method) + } + if got := r.Header.Get("traceparent"); got != traceparent { + t.Errorf("upstream traceparent = %q, want %q", got, traceparent) + } + return &http.Response{ + StatusCode: http.StatusTeapot, + Body: io.NopCloser(strings.NewReader("hello from upstream")), + Header: make(http.Header), + }, nil + })} + + payload, err := json.Marshal(fetchRequest{URL: "https://allowed.example/"}) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(payload))) + request.Header.Set("traceparent", traceparent) + newHandler(client).ServeHTTP(recorder, request) + + if recorder.Code != http.StatusTeapot { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusTeapot) + } + var got fetchResponse + if err := json.NewDecoder(recorder.Body).Decode(&got); err != nil { + t.Fatalf("decoding response: %v", err) + } + if got.StatusCode != http.StatusTeapot || got.Body != "hello from upstream" { + t.Errorf("response = %+v", got) + } +} + +func TestInvalidRequests(t *testing.T) { + tests := []struct { + name string + method string + body string + status int + }{ + {name: "method", method: http.MethodGet, body: `{}`, status: http.StatusMethodNotAllowed}, + {name: "malformed JSON", method: http.MethodPost, body: `{`, status: http.StatusBadRequest}, + {name: "missing hostname", method: http.MethodPost, body: `{"url":"https:///path"}`, status: http.StatusBadRequest}, + {name: "unsupported scheme", method: http.MethodPost, body: `{"url":"file:///etc/passwd"}`, status: http.StatusBadRequest}, + } + + handler := newHandler(http.DefaultClient) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(test.method, "/", strings.NewReader(test.body)) + handler.ServeHTTP(recorder, request) + if recorder.Code != test.status { + t.Errorf("status = %d, want %d; body = %s", recorder.Code, test.status, recorder.Body.String()) + } + }) + } +} + +func TestOutboundFailure(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("blocked") + })} + handler := newHandler(client) + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"url":"https://example.com/"}`)) + + handler.ServeHTTP(recorder, request) + + if recorder.Code != http.StatusBadGateway { + t.Errorf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh new file mode 100755 index 0000000000..271c550dde --- /dev/null +++ b/demos/egress/test-egress.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# End-to-end test for pluggable actor egress. Reproduces: +# * POSITIVE — a real, running Actor's plain-HTTP egress is transparently +# tunneled (nftables -> atunnel -> mTLS + CONNECT) through the Envoy egress +# gateway to an in-cluster target, and the gateway's ext_proc authenticates +# the actor certificate against the ate API (allowed, HTTP 200). +# * NEGATIVE — a pod holding a perfectly valid *pod* identity, but no actor +# certificate, cannot open a tunnel: the gateway's trusted_ca is the +# actor-identity CA, so the mTLS handshake itself is refused. +# +# Prerequisites: a substrate cluster with `--deploy-demo-egress` applied, plus +# kubectl and kubectl-ate on PATH. See demos/egress/README.md. +# +# Usage: +# demos/egress/test-egress.sh # run the tests +# demos/egress/test-egress.sh --cleanup # remove everything this script created + +set -o errexit -o nounset -o pipefail + +CTX="${KUBECTL_CONTEXT:-kind-kind}" +ATESPACE="${ATESPACE:-demo}" +ACTOR="${ACTOR:-egress-demo}" +TEMPLATE="${TEMPLATE:-ate-demo-egress/egress}" +TARGET_NS="${TARGET_NS:-egress-target}" +PROBE_POD="egress-identity-probe" + +K="kubectl --context ${CTX}" +KATE="kubectl-ate --context ${CTX}" + +log() { printf '\n\033[1;36m== %s\033[0m\n' "$*"; } +info() { printf ' %s\n' "$*"; } +pass() { printf '\033[1;32mPASS\033[0m %s\n' "$*"; } +fail() { printf '\033[1;31mFAIL\033[0m %s\n' "$*"; FAILED=1; } +FAILED=0 + +require() { command -v "$1" >/dev/null 2>&1 || { echo "missing required tool: $1"; exit 1; }; } + +cleanup() { + log "cleanup" + ${K} -n ate-system delete pod "${PROBE_POD}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + ${KATE} suspend actor "${ACTOR}" -a "${ATESPACE}" >/dev/null 2>&1 || true + ${KATE} delete actor "${ACTOR}" -a "${ATESPACE}" >/dev/null 2>&1 || true + ${K} delete namespace "${TARGET_NS}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + info "done" +} + +if [[ "${1:-}" == "--cleanup" ]]; then require kubectl; require kubectl-ate; cleanup; exit 0; fi + +require kubectl +require kubectl-ate +trap '[[ "${KEEP:-}" == "1" ]] || cleanup' EXIT + +log "preflight: egress gateway (Envoy + co-located ext_proc) is running" +${K} -n ate-system rollout status deployment/atenet-egress --timeout=120s + +log "deploy an in-cluster HTTP target (whoami)" +${K} create namespace "${TARGET_NS}" >/dev/null 2>&1 || true +${K} -n "${TARGET_NS}" create deployment whoami --image=traefik/whoami >/dev/null 2>&1 || true +${K} -n "${TARGET_NS}" expose deployment whoami --port=80 >/dev/null 2>&1 || true +${K} -n "${TARGET_NS}" rollout status deployment/whoami --timeout=120s +TARGET_IP=$(${K} -n "${TARGET_NS}" get svc whoami -o jsonpath='{.spec.clusterIP}') +info "target ClusterIP = ${TARGET_IP}" + +log "create + resume Actor ${ATESPACE}/${ACTOR}" +${KATE} create atespace "${ATESPACE}" >/dev/null 2>&1 || true +${KATE} create actor "${ACTOR}" -a "${ATESPACE}" --template "${TEMPLATE}" >/dev/null 2>&1 || true +${KATE} resume actor "${ACTOR}" -a "${ATESPACE}" >/dev/null 2>&1 || true +for _ in $(seq 1 30); do + ${KATE} get actors -a "${ATESPACE}" 2>/dev/null | grep -q "STATUS_RUNNING" && break + sleep 3 +done +${KATE} get actors -a "${ATESPACE}" 2>/dev/null | grep "${ACTOR}" || true +${KATE} get actors -a "${ATESPACE}" 2>/dev/null | grep -q "STATUS_RUNNING" || { echo "actor did not reach RUNNING"; exit 1; } + +egress_log_since() { ${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null | grep '\[egress\]' | tail -n +"$(( $1 + 1 ))"; } +egress_log_count() { ${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null | grep -c '\[egress\]' || true; } +# wait_egress_log : retry for log-shipping lag. +wait_egress_log() { for _ in $(seq 1 10); do if egress_log_since "$1" | grep -qE "$2"; then egress_log_since "$1" | grep -E "$2" | tail -1; return 0; fi; sleep 1; done; return 1; } + +############################################################################## +log "POSITIVE — real Actor egress is tunneled through the gateway (expect 200)" +############################################################################## +BEFORE=$(egress_log_count) +${K} -n ate-system port-forward service/atenet-router 18099:80 >/tmp/egress-pf.log 2>&1 & +PF=$!; sleep 4 +CODE=$(curl -s -o /tmp/egress-body.txt -w '%{http_code}' -X POST http://localhost:18099/ \ + -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H 'Content-Type: application/json' \ + -d "{\"url\":\"http://${TARGET_IP}:80/\"}" || true) +kill "${PF}" >/dev/null 2>&1 || true +sleep 1 +info "actor round-trip HTTP ${CODE}" +GW_IP=$(${K} -n ate-system get pod -l app=atenet-egress -o jsonpath='{.items[0].status.podIP}') +if [[ "${CODE}" == "200" ]]; then pass "actor fetched the target (HTTP 200)"; else fail "expected HTTP 200, got ${CODE}"; fi +if grep -q "RemoteAddr: ${GW_IP}" /tmp/egress-body.txt 2>/dev/null; then + pass "target saw the egress gateway (${GW_IP}) as its client — traffic went through the gateway" +else + info "target body RemoteAddr: $(grep -o 'RemoteAddr: [0-9.]*' /tmp/egress-body.txt 2>/dev/null || echo '?') (gateway IP ${GW_IP})" +fi +# The access log identifies the peer by its certificate SAN +# (spiffe://substrate-actor.local/atespace//actor/), not by any +# header the actor could have written. +if LINE=$(wait_egress_log "${BEFORE}" "actor/${ACTOR}.*code=200"); then + pass "gateway logged the CONNECT: ${LINE}" +else + fail "gateway did not log an allowed CONNECT for ${ACTOR}" +fi + +############################################################################## +log "NEGATIVE — a pod identity is not an actor identity (expect a refused handshake)" +############################################################################## +${K} apply -f - >/dev/null <<'YAML' +apiVersion: v1 +kind: Pod +metadata: + name: egress-identity-probe + namespace: ate-system +spec: + containers: + - name: curl + image: curlimages/curl:latest + command: ["sleep", "600"] + volumeMounts: + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev, readOnly: true } + volumes: + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: servicedns + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: { matchLabels: { podcert.ate.dev/canarying: live } } + path: trust-bundle.pem +YAML +${K} -n ate-system wait --for=condition=Ready pod/${PROBE_POD} --timeout=60s >/dev/null + +BEFORE=$(egress_log_count) +# %{http_connect} carries the proxy's CONNECT response code, and stays 000 when +# the tunnel never opens. curl exits non-zero on a failed handshake, so capture +# both and require that no CONNECT was ever answered. +PROBE=$(${K} -n ate-system exec ${PROBE_POD} -- sh -c "curl -s -o /dev/null -w '%{http_connect}' \ + --proxy-cacert /run/servicedns.podcert.ate.dev/trust-bundle.pem \ + --proxy-cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --proxy-key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --proxytunnel -x https://atenet-egress.ate-system.svc:443 http://${TARGET_IP}:80/; echo \" exit=\$?\"" || true) +CODE=${PROBE%% *} +info "pod-identity CONNECT attempt: http_connect=${CODE:-000}${PROBE#"${CODE}"}" +if [[ "${CODE}" != "200" ]]; then + pass "a pod identity cannot open an egress tunnel (no CONNECT succeeded)" +else + fail "expected the gateway to refuse a non-actor client certificate, but CONNECT returned 200" +fi +# A rejected handshake never becomes an HTTP request, so it produces no [egress] +# access-log line — only a connection-level TLS error. Report whatever the +# gateway logged so a real failure is diagnosable, but do not assert on it. +if LINE=$(egress_log_since "${BEFORE}" | tail -1); [[ -n "${LINE:-}" ]]; then + info "gateway egress log since the probe: ${LINE}" +else + info "no new [egress] access-log lines — the handshake was refused before HTTP, as expected" +fi + +echo +if [[ "${FAILED}" == "0" ]]; then + printf '\033[1;32mALL CHECKS PASSED\033[0m — pluggable egress + identity authentication working.\n' +else + printf '\033[1;31mSOME CHECKS FAILED\033[0m\n'; exit 1 +fi diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index 63124bfb4d..3c57b4d5a8 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -474,7 +474,7 @@ advertises the 4318 HTTP endpoint. use.** Envoy's tracer cluster is plaintext h2c, so an `https://` endpoint is neither honored nor silently downgraded: the router logs a warning, turns Envoy-side tracing off, and starts normally. Its own spans are unaffected. -Taking the xDS control plane down for every Envoy in the mesh over a tracing +Taking the xDS control plane down for every ingress Envoy over a tracing endpoint that works fine for the router's own exporter would be the larger failure. diff --git a/docs/roadmap.md b/docs/roadmap.md index 9663abc758..3d65845099 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -95,7 +95,7 @@ Below is a collection of finer-grained efforts which we believe align with the a * Agent Development Kit (ADK) Native Support: Developing first-class bindings for ADK, allowing developers to build stateful agents that natively leverage Substrate’s lifecycle management and persistent working memory. * LangChain Remote Execution Provider: A dedicated provider for LangChain to run complex, long-running agent tools in durable, sandboxed environments. * Native MCP Server Hosting: Built-in support for deploying Model Context Protocol (MCP) servers as managed Substrate Actors, creating a secure tool ecosystem for any LLM. -* Actor-to-Actor (A2A) Calling Model: Standardized protocol for actors to discover and call other actors within the mesh via the gateway. +* Actor-to-Actor (A2A) Calling Model: Standardized protocol for actors to discover and call other actors within Substrate via the gateway. * Native MCP Tool Hosting: Ability to define and deploy standard Model Context Protocol (MCP) servers as managed Substrate Actors, providing a plug-and-play ecosystem for agentic tools. ### Operability diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 5238e0a6b7..ca68115b1b 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -40,6 +40,7 @@ ATE_DEMOS=() # Include demos. source "${ROOT}"/hack/install-demo-counter.sh +source "${ROOT}"/hack/install-demo-egress.sh source "${ROOT}"/hack/install-demo-sandbox.sh source "${ROOT}"/hack/install-demo-claude-code-multiplex.sh source "${ROOT}"/hack/install-demo-multi-template.sh @@ -79,6 +80,7 @@ function usage() { echo "" echo " --create-jwt-authority-pool-secret Create JWT authority pool secret" echo " --create-actor-id-ca-pool-secret Create actor ID CA pool secret" + echo " --create-actor-id-ca-certs-secret Create actor ID CA certs secret" echo " --create-podcertificate-controller-cas Create podcertificate controller CAs" echo " --create-valkey-ca-certs-secret Create Valkey CA certs secret" echo " --create-api-server-env-vars Create ate-api-server env vars" @@ -222,10 +224,13 @@ apply_otel_config() { } # Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. +# The namespace defaults to the podcertificate controller's, where the signer +# CAs live; the actor-identity CA pool is in ate-system, so it passes its own. ca_pool_root_pem() { local secret="$1" + local namespace="${2:-podcertificate-controller-system}" local pool_json="" - pool_json=$(run_kubectl get secret -n podcertificate-controller-system "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) + pool_json=$(run_kubectl get secret -n "${namespace}" "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) local der_base64="" der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//') echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem @@ -275,6 +280,32 @@ create_actor_id_ca_pool_secret() { --secret-namespace=ate-system } +# The egress gateway has to verify actor client certificates, which means it +# needs the actor-identity CA root. actor-id-ca-pool Secret containts both +# root and CA signing key. This derives a cert-only Secret instead, following +# exactly the pattern create_valkey_ca_certs_secret already uses for the +# signer roots. +# +# TODO(liorlieberman): should this be published as ClusterTrustBundles? +create_actor_id_ca_certs_secret() { + log_step "create_actor_id_ca_certs_secret" + # Extract into its own variable first: errexit cannot see a substitution fail + # inside the create-secret argument list, which would silently produce an + # empty trust bundle and an egress gateway that rejects every actor. + local actorid_root="" + actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system) + if [[ -z "${actorid_root}" ]]; then + echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2 + return 1 + fi + + run_kubectl create secret generic actor-id-ca-certs \ + --from-literal=ca.crt="${actorid_root}" \ + -n ate-system \ + --dry-run=client -o yaml \ + | run_kubectl apply -f - +} + create_podcertificate_controller_cas() { log_step "create_podcertificate_controller_cas" run_kubectl create namespace podcertificate-controller-system || true @@ -405,6 +436,7 @@ deploy_ate_system() { run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout=120s run_kubectl rollout status deployment/ate-controller -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s + run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s run_kubectl rollout status statefulset/valkey-cluster -n ate-system --timeout=120s run_kubectl rollout status daemonset/atelet -n ate-system --timeout=120s } @@ -416,6 +448,9 @@ ensure_apiserver_prerequisites() { || create_jwt_authority_pool_secret run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ || create_actor_id_ca_pool_secret + # Derived from actor-id-ca-pool above, so it must come after it. + run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ + || create_actor_id_ca_certs_secret run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ || create_podcertificate_controller_cas run_kubectl get secret -n ate-system valkey-ca-certs >/dev/null 2>&1 \ @@ -476,11 +511,10 @@ deploy_atenet() { router_manifest="$(render_atenet_router_manifest)" echo "${router_manifest}" | run_kubectl apply -f - + run_ko apply -f manifests/ate-install/atenet-egress.yaml run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s - # The Deployment in atenet-dns.yaml is named "dns"; every other resource in - # that file is "atenet-dns". Waiting on the filename rather than the actual - # Deployment made this step fail with NotFound on every successful deploy. + run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s run_kubectl rollout status deployment/dns -n ate-system --timeout=120s } @@ -599,6 +633,8 @@ delete_atenet() { run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-router.yaml run_kubectl delete --ignore-not-found \ -f manifests/ate-install/components/agentgateway/configmap.yaml + run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-egress.yaml + run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-dns.yaml } deploy_benchmarks() { @@ -734,6 +770,7 @@ while [[ "$#" -gt 0 ]]; do --create-jwt-authority-pool-secret) create_jwt_authority_pool_secret ;; --create-actor-id-ca-pool-secret) create_actor_id_ca_pool_secret ;; + --create-actor-id-ca-certs-secret) create_actor_id_ca_certs_secret ;; --create-podcertificate-controller-cas) create_podcertificate_controller_cas ;; --create-valkey-ca-certs-secret) create_valkey_ca_certs_secret ;; --create-api-server-env-vars) create_api_server_env_vars ;; diff --git a/hack/install-demo-egress.sh b/hack/install-demo-egress.sh new file mode 100644 index 0000000000..e4e0074a9d --- /dev/null +++ b/hack/install-demo-egress.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is sourced as part of install-ate.sh. Do not run directly. + +ATE_DEMOS+=(demo-egress) # register demo-egress + +demo-egress_cmdline() { + case "${1}" in + --deploy-demo-egress) demo-egress_deploy ;; + --delete-demo-egress) demo-egress_delete ;; + *) + return 1 + ;; + esac + return 0 +} + +demo-egress_deploy() { + log_step "demo-egress_deploy" + ensure_crds + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" demos/egress/egress.yaml.tmpl \ + | run_ko apply -f - + + log_step "Waiting for egress demo to be ready..." + # The WorkerPool controller names the Deployment after the WorkerPool + # ("egress"), the same way demo-counter gets "deployment/counter". The old + # "egress-deployment" name was NotFound on every successful deploy. + run_kubectl rollout status deployment/egress -n ate-demo-egress --timeout=300s + run_kubectl wait --for=condition=Ready actortemplate/egress -n ate-demo-egress --timeout=300s +} + +demo-egress_delete() { + log_step "demo-egress_delete" + delete_demo_actors ate-demo-egress egress + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" demos/egress/egress.yaml.tmpl \ + | run_kubectl delete --ignore-not-found -f - +} diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh new file mode 100755 index 0000000000..19c69cfd94 --- /dev/null +++ b/hack/verify-egress-demo.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Preconditions: +# hack/create-kind-cluster.sh +# hack/install-ate-kind.sh --deploy-demo-egress +# +# The egress demo Actor accepts {"url":"..."} and performs an HTTP GET. With +# egress turned on, the Actor's outbound TCP is nftables-REDIRECTed +# into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the Envoy egress +# gateway, which terminates CONNECT and tunnels to the real destination. This +# script drives that path and shows the gateway's access log proving the actor's +# client certificate + CONNECT authority were seen. +set -o errexit -o nounset -o pipefail +ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}" + +CTX="${KUBECTL_CONTEXT:-kind-kind}" +K="kubectl --context ${CTX}" +ATESPACE="${ATESPACE:-demo}" +ACTOR="${ACTOR:-egress-demo}" +TARGET_URL="${TARGET_URL:-http://example.com/}" + +echo "== gateway should be running ==" +${K} -n ate-system rollout status deployment/atenet-egress --timeout=120s + +echo "== create atespace + actor ==" +kubectl-ate --context "${CTX}" create atespace "${ATESPACE}" 2>/dev/null || true +kubectl-ate --context "${CTX}" create actor "${ACTOR}" \ + --atespace "${ATESPACE}" --template ate-demo-egress/egress 2>/dev/null || true +${K} -n ate-system wait --for=condition=Ready "actor/${ACTOR}" 2>/dev/null || sleep 10 + +echo "== snapshot gateway log offset ==" +# -c envoy explicitly: the gateway pod also runs the ext-proc sidecar, and the +# [egress] access log belongs to Envoy. +BEFORE=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null | wc -l | tr -d ' ') + +echo "== drive actor egress: GET ${TARGET_URL} via the actor ==" +${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 & +PF=$!; trap 'kill ${PF} 2>/dev/null || true' EXIT +sleep 3 +RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ + -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H 'Content-Type: application/json' \ + -d "{\"url\":\"${TARGET_URL}\"}") || true +echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" + +echo "== NEW egress gateway access log lines (proof of CONNECT+mTLS+identity) ==" +# Envoy emits the CONNECT entry asynchronously (an external dst can land seconds +# after the actor's response), so we poll. And the Actor's HTTP client keeps the +# tunnel alive: a repeat fetch to a host it already reached rides the open tunnel +# and produces no new access-log entry at all, so a run against a warm actor +# would fail even though egress is working. Fall back to any tunnel already open +# for this actor's SAN before declaring failure. +SAN="atespace/${ATESPACE}/actor/${ACTOR}" +NEW="" +for _ in $(seq 1 15); do + NEW=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \ + | tail -n +"$((BEFORE + 1))" | grep '\[egress\]' || true) + [ -n "${NEW}" ] && break + sleep 2 +done +if [ -n "${NEW}" ]; then + echo "${NEW}" +elif ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \ + | grep -q "\[egress\].*${SAN}"; then + echo " no new CONNECT — the actor reused an already-open tunnel; its existing entries:" + ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 | grep "\[egress\].*${SAN}" | tail -3 +else + echo "!! no [egress] lines for ${SAN} — dumping recent gateway logs:" + ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=20 + exit 1 +fi +echo "== PASS: actor egress traversed the Envoy egress gateway ==" diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 058e588525..6951befd33 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -15,8 +15,10 @@ package e2e import ( + "bytes" "context" "fmt" + "io" "net/http" "time" @@ -31,7 +33,7 @@ const ( routerService = "atenet-router" ) -// RouterClient sends HTTP requests to actors through the atenet router, the +// RouterClient sends HTTP requests to actors through the ingress atenet-router, the // same way real traffic arrives (so the request is routed and, if needed, the // actor is resumed). It port-forwards the router Service, mirroring the // approach in internal/ateclient. @@ -41,7 +43,7 @@ type RouterClient struct { stop func() } -// NewRouterClient establishes a port-forward to the atenet router. Call Close +// NewRouterClient establishes a port-forward to the ingress atenet-router. Call Close // to tear it down. func NewRouterClient(ctx context.Context) (*RouterClient, error) { config, err := ateclient.LoadConfig(KubeConfig, KubeContext) @@ -70,13 +72,26 @@ func (c *RouterClient) Close() { c.stop() } -// Get issues GET path to actor through the router, setting the actor's mesh Host +// Get issues GET path to actor through the router, setting the actor's DNS Host // so the router routes (and resumes) it. The caller must close the body. func (c *RouterClient) Get(ctx context.Context, actorRef resources.ActorRef, path string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil) + return c.request(ctx, http.MethodGet, actorRef, path, nil) +} + +// PostJSON issues a POST with a JSON body to an Actor through the router. The +// caller must close the response body. +func (c *RouterClient) PostJSON(ctx context.Context, actorRef resources.ActorRef, path string, body []byte) (*http.Response, error) { + return c.request(ctx, http.MethodPost, actorRef, path, bytes.NewReader(body)) +} + +func (c *RouterClient) request(ctx context.Context, method string, actorRef resources.ActorRef, path string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) if err != nil { return nil, err } + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/json") + } // The router routes on the Host/:authority, not a header. req.Host = actorRef.DNSName() return c.http.Do(req) diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go new file mode 100644 index 0000000000..e963510847 --- /dev/null +++ b/internal/e2e/router_client_test.go @@ -0,0 +1,70 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/agent-substrate/substrate/internal/resources" +) + +func TestRouterClientPostJSON(t *testing.T) { + client := &RouterClient{ + baseURL: "http://router.test", + http: &http.Client{Transport: testRoundTripper(func(request *http.Request) (*http.Response, error) { + if request.Method != http.MethodPost { + t.Errorf("method = %q, want POST", request.Method) + } + if request.Host != "fetcher.demo.actors.resources.substrate.ate.dev" { + t.Errorf("host = %q", request.Host) + } + if request.URL.Path != "/fetch" { + t.Errorf("path = %q, want /fetch", request.URL.Path) + } + if request.Header.Get("Content-Type") != "application/json" { + t.Errorf("content type = %q, want application/json", request.Header.Get("Content-Type")) + } + body, err := io.ReadAll(request.Body) + if err != nil { + t.Fatalf("reading body: %v", err) + } + if string(body) != `{"url":"https://example.com/"}` { + t.Errorf("body = %q", body) + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("ok")), + Header: make(http.Header), + }, nil + })}, + } + + actorRef := resources.ActorRef{Atespace: "demo", Name: "fetcher"} + response, err := client.PostJSON(context.Background(), actorRef, "/fetch", []byte(`{"url":"https://example.com/"}`)) + if err != nil { + t.Fatalf("PostJSON: %v", err) + } + response.Body.Close() +} + +type testRoundTripper func(*http.Request) (*http.Response, error) + +func (f testRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index 1cb32d54b3..b4a919d9e6 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -29,9 +29,24 @@ import ( const networkingAtespace = "networking-e2e" +// actorTemplate identifies a demo ActorTemplate to build test Actors from, +// along with the hack/install-ate.sh flag that deploys it. +type actorTemplate struct { + namespace string + name string + // deployFlag names the install flag that creates the template, so a + // missing fixture reports how to fix it rather than just failing. + deployFlag string +} + +var ( + counterTemplate = actorTemplate{namespace: "ate-demo-counter", name: "counter", deployFlag: "--deploy-demo-counter"} + egressTemplate = actorTemplate{namespace: "ate-demo-egress", name: "egress", deployFlag: "--deploy-demo-egress"} +) + func TestActorDirectAccess(t *testing.T) { ctx := context.Background() - actorName, actor := createAndResumeActor(t, ctx, "direct") + actorName, actor := createAndResumeActor(t, ctx, "direct", counterTemplate) router := mustRouterClient(t, ctx) defer router.Close() @@ -68,7 +83,38 @@ func TestActorDirectAccess(t *testing.T) { }) } -func createAndResumeActor(t *testing.T, ctx context.Context, prefix string) (string, *ateapipb.Actor) { +// TestActorEgress exercises the full egress path. The Actor's outbound TCP +// connection is transparently redirected by nftables into atunnel, wrapped in +// mTLS with the Actor's own actor-identity certificate plus an HTTP CONNECT to +// atenet-egress, authorized there against that certificate, and only then +// dialed out. A masqueraded (pre-gateway) egress would also return 200, so this +// asserts the gateway is deployed and that it did not reject the Actor. +func TestActorEgress(t *testing.T) { + ctx := context.Background() + actorName, _ := createAndResumeActor(t, ctx, "egress", egressTemplate) + router := mustRouterClient(t, ctx) + defer router.Close() + + // The egress demo fetches the URL it is given and echoes the upstream + // status and body back. + payload := []byte(`{"url":"http://example.com/"}`) + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + response, err := router.PostJSON(ctx, actorRef, "/", payload) + if err != nil { + t.Fatalf("POST to egress Actor through ingress: %v", err) + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("reading egress response body (HTTP %d): %v", response.StatusCode, err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("Actor egress fetch returned HTTP %d, want 200; body: %s", response.StatusCode, body) + } + t.Logf("Actor egress fetch succeeded; body: %s", body) +} + +func createAndResumeActor(t *testing.T, ctx context.Context, prefix string, template actorTemplate) (string, *ateapipb.Actor) { t.Helper() clients := e2e.GetClients() actorName := fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano()) @@ -80,10 +126,10 @@ func createAndResumeActor(t *testing.T, ctx context.Context, prefix string) (str }) if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: networkingAtespace, Name: actorName}, - ActorTemplateNamespace: "ate-demo-counter", - ActorTemplateName: "counter", + ActorTemplateNamespace: template.namespace, + ActorTemplateName: template.name, }}); err != nil { - t.Fatalf("CreateActor: %v (deploy the fixture with --deploy-demo-counter)", err) + t.Fatalf("CreateActor from %s/%s: %v (deploy the fixture with %s)", template.namespace, template.name, err, template.deployFlag) } t.Cleanup(func() { _, _ = clients.SubstrateAPI.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef}) diff --git a/internal/resources/actorref.go b/internal/resources/actorref.go index 7d287ad8a5..0beabab26e 100644 --- a/internal/resources/actorref.go +++ b/internal/resources/actorref.go @@ -47,7 +47,7 @@ func (r ActorRef) LogValue() slog.Value { ) } -// DNSName returns the mesh DNS name the actor is reachable at. +// DNSName returns the uniform DNS name the actor is reachable at. // This is: "..actors.resources.substrate.ate.dev". func (r ActorRef) DNSName() string { return r.Name + "." + r.Atespace + "." + ActorDNSSuffix diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 560f32e204..c4c73a37d1 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -104,6 +104,12 @@ spec: - --actor-id-ca-pool=/run/actor-id-ca-pool/pool.json - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + # TODO(lior):Remove when egress API pr is merged. This turns on pluggable actor + # egress cluster-wide. ateapi stamps this address onto every atelet + # Run/Restore, ateom hands it to atunnel, and actor TCP egress is + # transparently redirected (nftables) into atunnel, which wraps it in + # mTLS + HTTP CONNECT to the egress gateway. + - --egress-gateway-address=atenet-egress.ate-system.svc:443 # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s - --drain-timeout=15s diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml new file mode 100644 index 0000000000..591ef31fcb --- /dev/null +++ b/manifests/ate-install/atenet-egress.yaml @@ -0,0 +1,388 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# contract expected by atunnel's egress client: +# * downstream mTLS on :443 (present servicedns identity, require + verify an +# actor-identity client cert), +# * terminate the actor's HTTP CONNECT and tunnel raw TCP to the CONNECT +# authority (the actor's original destination, always sent as IP:port). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: atenet-egress + namespace: ate-system +# +# No RBAC is bound to this ServiceAccount on purpose: the ext_proc sidecar runs +# with --mode=egress, which serves the egress ext_proc handler and nothing else +# — no xDS server and no ActorTemplate controller, so no Kubernetes access. It +# reaches the control plane over the ate API only. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: atenet-egress + namespace: ate-system +data: + envoy.yaml: | + admin: + address: + socket_address: { address: 0.0.0.0, port_value: 15000 } + static_resources: + listeners: + - name: egress + address: + socket_address: { address: 0.0.0.0, port_value: 443 } + filter_chains: + # Named so ext_proc can read it back as xds.filter_chain_name. Must + # match EgressFilterChainName in + # cmd/atenet/internal/router/extproc/dispatch.go. + - name: egress + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_certificates: + # Gateway server identity (servicedns signer). credential-bundle.pem + # holds the leaf cert and key concatenated. watched_directory picks + # up kubelet's projected certificate rotation without a restart. + - certificate_chain: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } + private_key: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } + watched_directory: { path: /run/servicedns.podcert.ate.dev } + validation_context: + # Actors authenticate with an actor-identity client cert. + # Envoy enforces chain, signature, and validity period here, so + # an unsigned or expired actor cert never reaches ext_proc; the + # ActorIdentity extension is checked there because Envoy cannot + # read custom X.509 extensions. + trusted_ca: { filename: /run/actor-id-ca-certs/ca.crt } + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: egress_connect + codec_type: HTTP1 + upgrade_configs: + - upgrade_type: CONNECT + # TODO(liorlieberman): Can we make this cleaner? + forward_client_cert_details: SANITIZE_SET + set_current_client_cert_details: + chain: true + # Emit the access log as soon as the CONNECT tunnel is established + # (atunnel keeps the tunnel open, so the default log-on-close would + # not fire during the demo). + access_log_options: + flush_log_on_tunnel_successfully_established: true + access_log: + - name: envoy.access_loggers.stdout + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + text_format_source: + inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" + route_config: + name: connect_route + virtual_hosts: + - name: connect + domains: ["*"] + routes: + - match: { connect_matcher: {} } + route: + cluster: egress_forward_proxy + upgrade_configs: + - upgrade_type: CONNECT + connect_config: {} + http_filters: + # Actor-identity authorization: on every egress CONNECT, ext_proc + # reads the actor certificate out of x-forwarded-client-cert, + # re-verifies it against the actor-identity CA, pulls the + # ActorIdentity extension, and asks the ate API whether that UID is + # a real, running actor. Denials come back as an immediate 403. + # Fails closed if the router is down. + - name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + grpc_service: + envoy_grpc: + cluster_name: ext_proc_server + timeout: 2s + failure_mode_allow: false + # How the ext_proc server tells egress from ingress. It applies + # opposite trust models to the two directions, and dispatches on + # this Envoy-asserted filter chain name so that no client can + # select the egress path by crafting a request. The value must + # match EgressFilterChainName in + # cmd/atenet/internal/router/extproc/dispatch.go; renaming the + # filter chain above without updating it fails closed (the + # request is classified as ingress, which this --mode=egress + # instance does not serve, and is refused with a 404). + request_attributes: + - xds.filter_chain_name + processing_mode: + request_header_mode: SEND + response_header_mode: SKIP + request_body_mode: NONE + response_body_mode: NONE + request_trailer_mode: SKIP + response_trailer_mode: SKIP + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + # ext_proc gRPC server = the atenet router, co-located in this pod as a + # sidecar and called over localhost (same topology the ingress gateway uses + # for its dataplane + ext_proc). + - name: ext_proc_server + type: STATIC + lb_policy: ROUND_ROBIN + connect_timeout: 1s + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: ext_proc_server + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 50051 + # Dials the terminated-CONNECT target (IP:port from the authority). atunnel + # always sends an IP:port, so DNS resolution is effectively a passthrough. + - name: egress_forward_proxy + lb_policy: CLUSTER_PROVIDED + connect_timeout: 5s + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: atenet-egress + namespace: ate-system + labels: + app: atenet-egress +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-egress + template: + metadata: + labels: + app: atenet-egress + spec: + serviceAccountName: atenet-egress + securityContext: + # Allow the non-root envoy user to bind :443. + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + terminationGracePeriodSeconds: 60 + containers: + - name: envoy + image: envoyproxy/envoy:v1.34-latest + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 65532 + args: + - -c + - /etc/envoy/envoy.yaml + - --service-node + - atenet-egress + - --service-cluster + - atenet-egress + # Prevents Envoy from fast-exiting on SIGTERM before the ext-proc + # sidecar finishes its drain sequence. Polls for the drain-complete + # marker the sidecar writes on the shared emptyDir, terminating Envoy + # as soon as the drain completes (or at terminationGracePeriodSeconds + # if the sidecar crashes). + # + # TODO(liorlieberman): decide the drain policy for long-lived CONNECT + # tunnels. envoyDrainer polls downstream_cx_active until it reaches + # zero, which suits ingress (short request/response connections) but + # not egress: atunnel holds a tunnel open for the life of the actor's + # connection, so an actor still streaming when the rollout starts keeps + # the count above zero until the ~15s window expires. Every rollout + # with active egress will therefore log "N downstream connections still + # active at the drain deadline". Either accept that as the honest + # outcome (we tried to drain, then cut), or give egress its own shorter + # window so we stop pretending a tunnel will close on its own. + lifecycle: + preStop: + exec: + command: ["sh", "-c", "while [ ! -f /var/run/atenet/drain-complete ]; do sleep 0.5; done"] + ports: + - name: https + containerPort: 443 + - name: admin + containerPort: 15000 + readinessProbe: + httpGet: + path: /ready + port: admin + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /ready + port: admin + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/envoy + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + readOnly: true + # Co-located ext_proc server: the same atenet router binary the ingress + # gateway runs, started with --mode=egress so it serves only the egress + # ext_proc handler (no xDS server, no ActorTemplate controller, no + # Kubernetes access). The egress Envoy calls it over localhost to + # authenticate actor identity against the ate API on every CONNECT. + # + # Ingress and egress are separate Deployments because they scale + # independently, not because they need separate binaries: one instance can + # serve both directions with --mode=all. + - name: ext-proc + image: ko://github.com/agent-substrate/substrate/cmd/atenet + args: + - router + - --mode=egress + - --namespace=ate-system + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address=dns:///api.ate-system.svc:443 + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + # Same bundle Envoy validates the handshake against. The handler + # re-verifies the chain in Go and reads the ActorIdentity extension out + # of it; without this flag the router has no actor-identity roots and + # denies every egress CONNECT with a 503. + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + # The egress Envoy's admin listener is on 15000, not the 9901 the flag + # defaults to (that is the ingress gateway's port). Without this the + # drain sequence dials a closed port, reads the connection refusal as + # "Envoy already exited", and reports a drain it never performed. + - --envoy-admin-address=127.0.0.1:15000 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + readinessProbe: + tcpSocket: + port: extproc + periodSeconds: 10 + volumeMounts: + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + # ext-proc's own client identity presented to ateapi. + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + # Actor-identity CA roots, for --actor-identity-ca-file above. + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + volumes: + - name: config + configMap: + name: atenet-egress + - name: drain-signal + emptyDir: {} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: atenet-egress + namespace: ate-system +spec: + type: ClusterIP + selector: + app: atenet-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index f29b6fd350..1dc2e3fe1f 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -161,6 +161,14 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atenet args: - "router" + # Serve the ingress direction only: the ingress ext_proc handler, plus + # the xDS server and ActorTemplate controller that configure the + # dataplane in this pod when it is Envoy (the agentgateway overlay is + # statically configured and starts neither). + # Egress runs the same binary with --mode=egress in its own + # Deployment (manifests/ate-install/atenet-egress.yaml), because the two + # directions scale independently. + - "--mode=ingress" - "--standalone" - "--namespace=ate-system" - "--port-http=8080" diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 6a6040ac2b..188bb87786 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -23,6 +23,7 @@ resources: - ../ate-controller.yaml - ../atelet.yaml - ../atenet-dns.yaml + - ../atenet-egress.yaml - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 5ce1791413..107c8950fb 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -34,15 +34,15 @@ patches: namespace: ate-system patch: |- - op: test - path: /spec/template/spec/containers/0/args/4 + path: /spec/template/spec/containers/0/args/5 value: --port-xds=18000 - op: remove - path: /spec/template/spec/containers/0/args/4 + path: /spec/template/spec/containers/0/args/5 - op: test - path: /spec/template/spec/containers/0/args/8 + path: /spec/template/spec/containers/0/args/9 value: --envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem - op: remove - path: /spec/template/spec/containers/0/args/8 + path: /spec/template/spec/containers/0/args/9 - op: test path: /spec/template/spec/containers/0/ports/0/name value: xds diff --git a/manifests/ate-install/kind-token-client/kustomization.yaml b/manifests/ate-install/kind-token-client/kustomization.yaml index c7db1cc3a4..d95c6be060 100644 --- a/manifests/ate-install/kind-token-client/kustomization.yaml +++ b/manifests/ate-install/kind-token-client/kustomization.yaml @@ -53,10 +53,10 @@ patches: namespace: ate-system patch: |- - op: test - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem - op: remove - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-use-token-auth=true diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 54a6b0b33d..c6f9e5e51e 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -26,6 +26,7 @@ resources: - ../ate-controller.yaml - ./atelet - ../atenet-dns.yaml + - ../atenet-egress.yaml - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml diff --git a/manifests/ate-install/token-client/kustomization.yaml b/manifests/ate-install/token-client/kustomization.yaml index 3b172a17bd..fc39fff8ab 100644 --- a/manifests/ate-install/token-client/kustomization.yaml +++ b/manifests/ate-install/token-client/kustomization.yaml @@ -53,10 +53,10 @@ patches: namespace: ate-system patch: |- - op: test - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem - op: remove - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-use-token-auth=true