egress: add egress gateway support - atenet-egress

.github/workflows/pr-workflow.yaml

@@ -96,6 +96,10 @@ jobs:
       run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }}
     - name: Deploy gVisor counter demo
       run: hack/install-ate-kind.sh --deploy-demo-counter
+    - name: Deploy egress demo
+      # TestActorEgress in the networking suite builds its Actor from the
+      # ate-demo-egress/egress ActorTemplate, so the fixture has to exist before.
+      run: hack/install-ate-kind.sh --deploy-demo-egress
     - name: Wait for micro-VM golden snapshot
       run: |
         kubectl --context kind-kind wait --for=condition=Ready \

Makefile

@@ -57,7 +57,7 @@ build-atenet:
 
 .PHONY: build-demos
 build-demos:
-	$(KO) build --ldflags="$(LDFLAGS)" ./demos/counter
+	$(KO) build --ldflags="$(LDFLAGS)" ./demos/counter ./demos/egress
 
 .PHONY: test
 test:

cmd/atenet/internal/router/README.md

@@ -7,7 +7,7 @@ Router has several responsibilities:
   Services in Kubernetes.
   With `--atenet-router=agentgateway`, the sidecar uses a static ConfigMap and
   atenet does not start an xDS server.
-* ext_proc server for the proxy. To make the deployment and debugging easier, we will run this component together
+* ext_proc server for the dataplane. To make the deployment and debugging easier, we will run this component together
   with the router, but this will be split later into its own component.
   * ext_proc will call into the ATE gRPC API to get the set of relevant backends (specific the worker IP) and
     route the traffic accordingly
@@ -20,12 +20,60 @@ Router has several responsibilities:
   bounded wait elapses, instead of failing fast. See
   [docs/request-parking.md](../../../../../docs/request-parking.md).
 * Drains gracefully on SIGTERM: flips `/readyz` so the Service stops sending
-  new connections, waits out endpoint propagation (`--drain-delay`), drives
-  Envoy's admin API to drain established connections, gracefully stops the
-  ext_proc server so parked requests finish normally (`--drain-timeout`,
-  derived from the parking budget), then writes a drain-complete marker that
-  releases the Envoy container's `preStop` hook. See `drain.go` and
-  `envoydrain.go`.
+  new connections, waits out endpoint propagation (`--drain-delay`), drains the
+  dataplane's established connections (Envoy only — driven over its admin API;
+  agentgateway manages its own termination), gracefully stops the ext_proc
+  server so parked requests finish normally (`--drain-timeout`, derived from
+  the parking budget), then writes a drain-complete marker that releases the
+  dataplane container's `preStop` hook. See `drain.go` and `envoydrain.go`.
+* Authenticates actor identity on egress: on every CONNECT, the egress
+  gateway's ext_proc handler re-verifies the actor's client certificate against
+  the actor-identity CA, reads the `ActorIdentity` X.509 extension out of it,
+  and checks the certified UID against the ATE API.
+
+## packages
+
+The ext_proc server handles both traffic directions, and they apply opposite
+trust models — egress derives the actor identity from a client certificate the
+gateway verified against the actor-identity CA, ingress treats every request
+header as unauthenticated client input — so the two are kept in separate
+packages that cannot reach into each other:
+
+* `extproc` — the mux, and nothing else. It terminates the ext_proc stream,
+  decides which direction a request arrived on, dispatches to the `Handler`
+  registered for that direction, and records latency and outcome. It also owns
+  the vocabulary both handlers share (`RequestMetadata`, `Result`, `ReqError`).
+  It imports neither handler package.
+* `ingress` — resume, park, and route to the actor's worker.
+* `egress` — certificate-based actor-identity authentication for outbound
+  CONNECTs.
+
+Direction is decided by the filter chain the dataplane says accepted the
+request (`xds.filter_chain_name`, an Envoy attribute the egress gateway is
+configured to send), never by anything in the request itself, so a client
+cannot pick the egress path by crafting one. `router` itself does the wiring.
+
+## modes
+
+One binary serves both directions. `--mode` selects which:
+
+| `--mode` | ext_proc handlers | xDS server + ActorTemplate controller | Kubernetes access |
+| --- | --- | --- | --- |
+| `ingress` | ingress | yes | yes |
+| `egress` | egress | no | none |
+| `all` (default) | both | yes | yes |
+
+The mux refuses a direction this instance was not started to serve (404) rather
+than falling back to the other handler, which would run the request through the
+wrong trust model.
+
+Ingress and egress are deployed separately today — `atenet-router` fronts the
+ingress dataplane, `atenet-egress` the egress gateway — because the two scale
+independently, not because they need separate binaries.
+
+The `--atenet-router` choice only applies to the ingress dataplane. The egress
+gateway is its own Deployment with a statically configured Envoy, so
+`--atenet-router=agentgateway` leaves it untouched.
 
 ## status page
 

cmd/atenet/internal/router/cmd.go

@@ -20,14 +20,16 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+
+	"github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress"
 )
 
 func NewRouterCmd() *cobra.Command {
 	var cfg routerConfig
 
 	cmd := &cobra.Command{
 		Use:   "router",
-		Short: "Router components including xDS server and Envoy ExtProc gateway processing server",
+		Short: "Router components including the Envoy xDS server and the ext_proc gateway processing server",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			srv, err := NewRouterServer(cfg)
 			if err != nil {
@@ -39,6 +41,7 @@ func NewRouterCmd() *cobra.Command {
 		},
 	}
 
+	cmd.Flags().StringVar((*string)(&cfg.Mode), "mode", string(ModeAll), fmt.Sprintf("Traffic direction this instance serves: %q (also runs the ingress control plane — the xDS server and ActorTemplate controller — for an Envoy dataplane), %q (ext_proc only, needs no Kubernetes access), or %q for both. The ext_proc mux refuses a direction this instance was not started to serve rather than falling back to the other one", ModeIngress, ModeEgress, ModeAll))
 	cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error")
 	cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.")
 	cmd.Flags().BoolVar(&cfg.Standalone, "standalone", false, "Run in standalone mode, bypassing creation of managed deployment and services in Kubernetes cluster")
@@ -48,17 +51,18 @@ func NewRouterCmd() *cobra.Command {
 	cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.")
 	cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router")
 	cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections")
-	cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server")
-	cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the Envoy External Processing (ext_proc) server")
+	cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the External Processing (ext_proc) server the dataplane calls")
+	cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the External Processing (ext_proc) server")
 	cmd.Flags().StringVar(&cfg.EnvoyImage, "envoy-image", "envoyproxy/envoy:v1.30-latest", "Image URI used for dynamically launched router instances")
 	cmd.Flags().StringVar(&cfg.TemplatesFile, "actor-templates-file", "", "Path to offline YAML configuration file listing ActorTemplates")
 	cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)")
 	cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services")
-	cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router")
+	cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the router dataplane")
 	cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.")
 	cmd.Flags().StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80).")
 	cmd.Flags().StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.")
 	cmd.Flags().StringVar(&cfg.UpstreamSpiffePrefix, "upstream-spiffe-prefix", "spiffe://cluster.local/", "SPIFFE URI SAN prefix (trust domain) the actor's atunnel server cert must match. Empty falls back to default SAN check against the dialed pod IP (which SPIFFE-only certs never match).")
+	cmd.Flags().StringVar(&cfg.ActorIdentityCAFile, "actor-identity-ca-file", "", "PEM trust bundle for the actor-identity CA, used to verify the actor client certificates presented on egress CONNECTs. Required by the egress gateway's ext_proc sidecar; empty (the default) leaves egress authentication unconfigured and every egress CONNECT is denied.")
 	// Envoy learns the collector over xDS rather than from its own environment,
 	// so the router has to carry the address for it. Defaulting to
 	// OTEL_EXPORTER_OTLP_ENDPOINT — the same variable the router's own exporter
@@ -70,19 +74,19 @@ func NewRouterCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&cfg.Auth.AteapiUseTokenAuth, "ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.")
 	cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.")
 	cmd.Flags().DurationVar(&cfg.RouteTimeout, "route-timeout", defaultRouteTimeout, "Envoy's end-to-end timeout on the workload route, bounding one request from the ingress listener to the actor's response. Raise it for actors whose turns legitimately run long — a harness relaying an LLM completion holds the request open for the whole generation. This does not cover the resume that may precede the request; see --parked-request-budget")
-	cmd.Flags().DurationVar(&cfg.ParkedRequest.Budget, "parked-request-budget", defaultParkedRequestBudget, "Maximum time a resume flight keeps a request parked (held and retried) waiting for its actor to become routable; concurrent requests for the same actor share one flight and its budget")
-	cmd.Flags().IntVar(&cfg.ParkedRequest.Max, "parked-request-max", defaultParkedRequestMax, "Maximum number of requests that may be parked simultaneously; excess requests are shed with 503. 0 disables parking (requests fail fast on worker-pool saturation)")
-	cmd.Flags().DurationVar(&cfg.ParkedRequest.RetryInterval, "parked-request-retry-interval", defaultParkedRequestRetryInterval, "Delay before a parked request's first resume retry")
-	cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryFactor, "parked-request-retry-factor", defaultParkedRequestRetryFactor, "Multiplier applied to the retry delay after each attempt; must be >= 1")
-	cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryJitter, "parked-request-retry-jitter", defaultParkedRequestRetryJitter, "Random fraction in [0, 1) added to each retry delay to de-synchronize parked requests")
+	cmd.Flags().DurationVar(&cfg.ParkedRequest.Budget, "parked-request-budget", ingress.DefaultParkedRequestBudget, "Maximum time a resume flight keeps a request parked (held and retried) waiting for its actor to become routable; concurrent requests for the same actor share one flight and its budget")
+	cmd.Flags().IntVar(&cfg.ParkedRequest.Max, "parked-request-max", ingress.DefaultParkedRequestMax, "Maximum number of requests that may be parked simultaneously; excess requests are shed with 503. 0 disables parking (requests fail fast on worker-pool saturation)")
+	cmd.Flags().DurationVar(&cfg.ParkedRequest.RetryInterval, "parked-request-retry-interval", ingress.DefaultParkedRequestRetryInterval, "Delay before a parked request's first resume retry")
+	cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryFactor, "parked-request-retry-factor", ingress.DefaultParkedRequestRetryFactor, "Multiplier applied to the retry delay after each attempt; must be >= 1")
+	cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryJitter, "parked-request-retry-jitter", ingress.DefaultParkedRequestRetryJitter, "Random fraction in [0, 1) added to each retry delay to de-synchronize parked requests")
 	cmd.Flags().IntVar(&cfg.ExtProcMaxRequests, "extproc-max-requests", 0, "Circuit-breaker max_requests for Envoy's ext_proc cluster; 0 (the default) derives it as twice --parked-request-max (minimum 1024). Explicit values must be >= --parked-request-max: every parked request holds one slot for its full wait, and the excess is fast-path headroom")
 	// Graceful shutdown knobs. The router sits behind a Service, so
 	// route-drain window is needed: after SIGTERM the readiness flip
 	// must propagate to the Service endpoints before the drain starts.
 	cmd.Flags().DurationVar(&cfg.DrainDelay, "drain-delay", 13*time.Second, "How long to keep serving after SIGTERM before starting the drain, covering readiness-probe detection and Service endpoint propagation")
 	cmd.Flags().DurationVar(&cfg.DrainTimeout, "drain-timeout", 0, "Deadline for the ext_proc drain on shutdown; streams still open past it (parked requests included) are forcefully cancelled. 0 (the default) derives --parked-request-budget + the actor route timeout + margin so parked requests always finish normally. Explicit values must be >= --parked-request-budget")
 	cmd.Flags().StringVar(&cfg.EnvoyAdminAddr, "envoy-admin-address", "127.0.0.1:9901", "Envoy admin interface the shutdown sequence drives to drain the sidecar (healthcheck/fail, drain_listeners, stats polling). Ignored with --atenet-router=agentgateway")
-	cmd.Flags().StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the Envoy container's preStop hook polls for it so Envoy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake")
+	cmd.Flags().StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the dataplane container's preStop hook polls for it so the proxy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake")
 
 	return cmd
 }