From 9220b82bcaed78089efa91371ec59b9dfeafe375 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Mon, 27 Jul 2026 16:23:31 -0700 Subject: [PATCH 1/6] egress: add an Envoy egress gateway. The gateway terminates actor's CONNECT request. It requires downstream mTLS, so only a worker's atunnel can reach it. The gateway consists of an Envoy and an `atenet router --standalone` ext_proc sidecar. --- cmd/atenet/internal/router/cmd.go | 1 + cmd/atenet/internal/router/config.go | 13 +- cmd/atenet/internal/router/extproc.go | 42 +- cmd/atenet/internal/router/extproc_egress.go | 451 +++++++++++ .../internal/router/extproc_egress_test.go | 698 ++++++++++++++++++ cmd/atenet/internal/router/extproc_in.go | 6 + cmd/atenet/internal/router/extproc_test.go | 8 +- cmd/atenet/internal/router/router.go | 17 +- cmd/atenet/internal/router/status_test.go | 2 +- hack/install-ate.sh | 60 +- manifests/ate-install/ate-api-server.yaml | 9 + manifests/ate-install/ateway-egress.yaml | 393 ++++++++++ manifests/ate-install/base/kustomization.yaml | 1 + manifests/ate-install/kind/kustomization.yaml | 1 + 14 files changed, 1681 insertions(+), 21 deletions(-) create mode 100644 cmd/atenet/internal/router/extproc_egress.go create mode 100644 cmd/atenet/internal/router/extproc_egress_test.go create mode 100644 manifests/ate-install/ateway-egress.yaml diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 4c199aab1..374bb11b4 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -59,6 +59,7 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80).") cmd.Flags().StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") cmd.Flags().StringVar(&cfg.UpstreamSpiffePrefix, "upstream-spiffe-prefix", "spiffe://cluster.local/", "SPIFFE URI SAN prefix (trust domain) the actor's atunnel server cert must match. Empty falls back to default SAN check against the dialed pod IP (which SPIFFE-only certs never match).") + cmd.Flags().StringVar(&cfg.ActorIdentityCAFile, "actor-identity-ca-file", "", "PEM trust bundle for the actor-identity CA, used to verify the actor client certificates presented on egress CONNECTs. Required by the egress gateway's ext_proc sidecar; empty (the default) leaves egress authentication unconfigured and every egress CONNECT is denied.") // Envoy learns the collector over xDS rather than from its own environment, // so the router has to carry the address for it. Defaulting to // OTEL_EXPORTER_OTLP_ENDPOINT — the same variable the router's own exporter diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 833ea4630..d7472e8f6 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -67,8 +67,17 @@ type routerConfig struct { // UpstreamSpiffePrefix validates the actor's atunnel server cert by its // SPIFFE URI SAN prefix (trust domain) instead of the dialed pod IP. UpstreamSpiffePrefix string - LogLevel string - MetricsAddr string + + // ActorIdentityCAFile is the PEM trust bundle for the actor-identity CA, + // used by the egress gateway's ext_proc sidecar to verify the actor client + // certificates atunnel presents on egress CONNECTs. Only that deployment + // sets it; empty leaves egress authentication unconfigured, which makes + // every egress CONNECT fail closed and is correct for an ingress-only + // router (it never sees the egress listener). + ActorIdentityCAFile string + + LogLevel string + MetricsAddr string // OtlpCollectorAddress is the OTLP gRPC collector that Envoy reports // tracing spans to, as host:port or an http:// URL. It defaults to // OTEL_EXPORTER_OTLP_ENDPOINT — Envoy gets its whole configuration over diff --git a/cmd/atenet/internal/router/extproc.go b/cmd/atenet/internal/router/extproc.go index a668a813d..1094a98c0 100644 --- a/cmd/atenet/internal/router/extproc.go +++ b/cmd/atenet/internal/router/extproc.go @@ -16,6 +16,7 @@ package router import ( "context" + "crypto/x509" "errors" "fmt" "io" @@ -47,17 +48,23 @@ type ExtProcServer struct { routeDuration metric.Float64Histogram parking *parkingLot routeViaAuthority bool + // actorIdentityRoots are the actor-identity CA certificates the egress + // handler verifies actor client certificates against. Only the ext_proc + // sidecar of the egress gateway sets it; nil makes every egress CONNECT + // fail closed and is the correct state for an ingress-only router. + actorIdentityRoots *x509.CertPool } -func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool) *ExtProcServer { +func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool, actorIdentityRoots *x509.CertPool) *ExtProcServer { return &ExtProcServer{ - port: port, - apiClient: apiClient, - recorder: NewQueryRecorder(100), - resumer: NewActorResumer(apiClient, withParking(parkCfg)), - routeDuration: routeDuration, - parking: newParkingLot(parkCfg, parkMetrics), - routeViaAuthority: routeViaAuthority, + port: port, + apiClient: apiClient, + recorder: NewQueryRecorder(100), + resumer: NewActorResumer(apiClient, withParking(parkCfg)), + routeDuration: routeDuration, + parking: newParkingLot(parkCfg, parkMetrics), + routeViaAuthority: routeViaAuthority, + actorIdentityRoots: actorIdentityRoots, } } @@ -88,7 +95,24 @@ func (s *ExtProcServer) Process(stream extprocv3.ExternalProcessor_ProcessServer switch reqType := req.Request.(type) { case *extprocv3.ProcessingRequest_RequestHeaders: start := time.Now() - hResponse, rqm, target, tmplNs, tmplName, resumeOutcome, err := s.handleRequestHeaders(stream.Context(), reqType.RequestHeaders) + // One ext_proc server handles both directions: actor egress + // CONNECT requests and ingress requests. Which one is decided by + // the accepting listener, not by anything in the request itself + // (see isEgressRequest). + // + // SEE(lior): main grew a ResumeOutcome return on + // handleRequestHeaders while this branch was out. Rather than + // splitting the dispatch into two separately-typed call sites, the + // egress handler was widened to the same signature and returns + // ResumeOutcomeNone — egress requires an already-RUNNING actor and + // never resumes one, so "none" is accurate, and it keeps the + // route-duration metric's resume label a closed set (no new empty + // value) across both directions. + handle := s.handleRequestHeaders + if isEgressRequest(req) { + handle = s.handleEgressRequestHeaders + } + hResponse, rqm, target, tmplNs, tmplName, resumeOutcome, err := handle(stream.Context(), reqType.RequestHeaders) elapsed := time.Since(start) outcomeStr := classifyOutcome(err) resumeStr := string(resumeOutcome) diff --git a/cmd/atenet/internal/router/extproc_egress.go b/cmd/atenet/internal/router/extproc_egress.go new file mode 100644 index 000000000..9649a6a2e --- /dev/null +++ b/cmd/atenet/internal/router/extproc_egress.go @@ -0,0 +1,451 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "log/slog" + "net/url" + "slices" + "strings" + "time" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +const ( + // EgressListenerName is the Envoy listener that terminates actor egress + // CONNECTs. It must stay in sync with the listener name in + // manifests/ate-install/ateway-egress.yaml. + EgressListenerName = "egress" + // ListenerNameAttribute is the CEL attribute carrying the name of the + // listener that accepted the request. The egress Envoy asks for it via + // request_attributes on its ext_proc filter. + ListenerNameAttribute = "xds.listener_name" + + // forwardedClientCertHeader is the header Envoy fills in with details of + // the mTLS peer, including the PEM chain it validated. The egress listener + // sets forward_client_cert_details: SANITIZE_SET, so whatever a client + // sends under this name is discarded and replaced by Envoy's own value. + // + // This is the only channel that can carry a whole certificate to ext_proc: + // the CEL request attributes Envoy exposes (subject, SANs, SHA-256 digest) + // cannot express the custom ActorIdentity X.509 extension this gateway + // authorizes on. + forwardedClientCertHeader = "x-forwarded-client-cert" + // xfccChainKey is the x-forwarded-client-cert key holding the URL-encoded + // PEM of the full presented chain, leaf first. + xfccChainKey = "chain" +) + +// isEgressRequest reports whether an ext_proc RequestHeaders callback arrived on +// the egress gateway's listener rather than on an ingress listener. This lets +// one ext_proc server handle both directions off the same stream. +// +// Dispatch is by listener, not by :method, because the two handlers apply +// opposite trust models: on egress the actor identity comes from a client +// certificate Envoy validated, while on ingress every request header is +// unauthenticated client input. Keying on :method would let any external client +// sending CONNECT select the egress handler and use its denial messages as an +// actor-existence and status oracle. Envoy asserts the listener name; the +// request cannot influence it. +// +// An unrecognized or absent attribute means ingress, the fail-safe direction: an +// egress request misrouted to the ingress handler fails to parse as an actor DNS +// name and 404s, whereas the reverse leaks control-plane state. +func isEgressRequest(req *extprocv3.ProcessingRequest) bool { + return listenerName(req) == EgressListenerName +} + +// listenerName returns the xds.listener_name attribute Envoy attached to the +// request, or "" when the listener did not request the attribute. The +// attributes map is keyed by the ext_proc filter's name within the HCM chain, +// which we do not want to hardcode here, so scan every entry. +func listenerName(req *extprocv3.ProcessingRequest) string { + for _, attrs := range req.GetAttributes() { + if v, ok := attrs.GetFields()[ListenerNameAttribute]; ok { + return v.GetStringValue() + } + } + return "" +} + +// handleEgressRequestHeaders authenticates the actor behind an egress CONNECT +// before the gateway tunnels it out, using the actor certificate atunnel +// presented in the mTLS handshake. Nothing the actor can write — no CONNECT +// header, no request metadata — contributes to the identity; the only inputs +// are the certificate the actor-identity CA signed and the control plane's own +// view of that actor. +// +// Authorization by destination and credential/token injection are deliberately +// a TODO once we have SessionIdentity RPC service figured out. +// +// The signature mirrors handleRequestHeaders so Process can dispatch to either +// with a single branch. The (target, tmplNs, tmplName) results are unused for +// egress and returned empty. +// +// SEE(lior): the trailing ResumeOutcome exists only to match +// handleRequestHeaders after main added it. Egress never resumes an actor — it +// requires one already RUNNING — so every path returns ResumeOutcomeNone. +func (s *ExtProcServer) handleEgressRequestHeaders( + ctx context.Context, + reqHeaders *extprocv3.HttpHeaders, +) (*extprocv3.HeadersResponse, *requestMetadata, string, string, string, ResumeOutcome, error) { + metadata := newRequestMetadata(reqHeaders.Headers.GetHeaders()) + + // Dispatch is by listener, so reaching here means the egress listener + // accepted the request. That listener only routes CONNECT (its sole route + // is a connect_matcher), so anything else is a config drift rather than a + // client the gateway should tunnel for. + if !strings.EqualFold(metadata.method, "CONNECT") { + return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_MethodNotAllowed, + "egress denied: expected CONNECT, got %q", metadata.method) + } + + // No roots means the gateway cannot authenticate anyone. Fail closed, and + // as 503 rather than 403: this is our misconfiguration, not the actor's. + if s.actorIdentityRoots == nil { + return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_ServiceUnavailable, + "egress unavailable: no actor-identity CA configured") + } + + identity, err := s.authenticateActorCertificate(metadata) + if err != nil { + // The message stays generic on purpose: an actor that fails + // authentication has not proven it is anyone, so it gets no detail + // about why. The specific reason is logged below instead. + slog.WarnContext(ctx, "egress denied: actor certificate rejected", slog.Any("err", err)) + return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + "egress denied: invalid actor certificate") + } + + atespace := identity.Atespace + actorName := identity.ActorName + actorUID := identity.ActorUid + // For a CONNECT the :authority is the actor's original destination (IP:port). + destination := metadata.host + + // The CA only ever mints these from control-plane state, so a name that is + // not a legal resource name means the CA or its inputs are compromised. + if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) { + return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + "egress denied: invalid actor identity %q/%q", atespace, actorName) + } + + // Confirm the certified actor still exists. The name is only a lookup key + // here; the UID below is what actually authorizes. + actor, err := s.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: actorName}, + }) + if err != nil { + return nil, metadata, "", "", "", ResumeOutcomeNone, mapEgressIdentityError(atespace, actorName, err) + } + + // Authorize on the UID, not the name. Names are reused: delete an actor and + // recreate it under the same atespace/name and it is a different actor with + // a different UID. A certificate outliving its actor must not carry over to + // the successor, so the UID the CA certified has to match the UID the + // control plane holds right now. + if uid := actor.GetMetadata().GetUid(); uid != actorUID { + slog.WarnContext(ctx, "egress denied: actor UID mismatch", + slog.String("atespace", atespace), + slog.String("actor", actorName), + slog.String("certificateActorUid", actorUID), + slog.String("currentActorUid", uid)) + return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + "egress denied: actor %q/%q is not the actor this certificate was issued to", atespace, actorName) + } + + // The actor performing egress must actually be running. + if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING { + return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + "egress denied: actor %q/%q is %s, not running", atespace, actorName, actor.GetStatus()) + } + + slog.InfoContext(ctx, "egress identity authenticated", + slog.String("atespace", atespace), + slog.String("actor", actorName), + slog.String("actorUid", actorUID), + slog.String("destination", destination), + slog.String("status", actor.GetStatus().String())) + + // Identity is authenticated; let the CONNECT proceed unchanged. Milestone 2 + // would additionally authorize `destination` and inject upstream credentials + // here by returning a HeaderMutation. + return &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, metadata, "", "", "", ResumeOutcomeNone, nil +} + +// authenticateActorCertificate turns the mTLS peer certificate Envoy recorded +// on the request into a verified ActorIdentity, or an error describing why it +// cannot be trusted. +func (s *ExtProcServer) authenticateActorCertificate(metadata *requestMetadata) (*substratex509.ActorIdentity, error) { + header := metadata.headers[forwardedClientCertHeader] + if header == "" { + return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader) + } + chain, err := parseXFCCChain(header) + if err != nil { + return nil, err + } + return s.verifyActorCertificate(chain) +} + +// verifyActorCertificate checks that chain[0] is a live, non-CA, client-auth +// actor certificate issued by the actor-identity CA, and returns the single +// ActorIdentity it carries. +// +// ########################################################################### +// SEE(lior): READ THIS IF YOU ARE WONDERING WHY WE VERIFY THE CHAIN TWICE. +// +// The egress listener already does full mTLS: require_client_certificate with +// the actor-identity CA as its trusted_ca, so Envoy refuses the handshake for +// anything this function would also reject on chain, expiry, or signature. The +// re-verification below is therefore redundant *today*, and it is here on +// purpose: +// +// - Envoy validates the chain but cannot look at the ActorIdentity +// extension. This function has to parse the certificate regardless, and +// parsing an unverified certificate and then trusting its contents is the +// failure mode that keeps producing CVEs. Verifying what we parse keeps the +// trust decision in one place instead of split across a YAML file and a Go +// file. +// - It makes the handler safe under Envoy config drift. Someone relaxing +// require_client_certificate, widening trusted_ca, or putting another proxy +// in front should not silently turn this into an unauthenticated endpoint. +// - It costs one signature verification per CONNECT, not per request: the +// tunnel is established once and then carries raw TCP. +// +// If you decide the Envoy-side check is authoritative and this is dead weight, +// this function is the thing to delete — but keep the ActorIdentity extraction +// and the IsCA/EKU/purpose checks below it, because Envoy does none of those. +// ########################################################################### +func (s *ExtProcServer) verifyActorCertificate(chain []*x509.Certificate) (*substratex509.ActorIdentity, error) { + leaf := chain[0] + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + intermediates.AddCert(cert) + } + + now := time.Now() + if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) { + return nil, fmt.Errorf("actor certificate is outside its validity period (%s..%s)", + leaf.NotBefore.Format(time.RFC3339), leaf.NotAfter.Format(time.RFC3339)) + } + // An actor certificate is an end-entity credential. Refusing IsCA here stops + // a leaked or mis-issued CA certificate from being replayed as a leaf: chain + // verification alone would happily accept one. + if leaf.IsCA { + return nil, fmt.Errorf("actor certificate is a CA certificate") + } + // Require ClientAuth explicitly rather than relying on VerifyOptions.KeyUsages: + // an empty ExtKeyUsage means "any usage" to crypto/x509 and would pass. This + // mirrors the check atunnel makes on the certificate when it mints it. + if !slices.Contains(leaf.ExtKeyUsage, x509.ExtKeyUsageClientAuth) { + return nil, fmt.Errorf("actor certificate cannot authenticate a TLS client") + } + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: s.actorIdentityRoots, + Intermediates: intermediates, + CurrentTime: now, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + return nil, fmt.Errorf("actor certificate is not signed by the actor-identity CA: %w", err) + } + + // ActorIdentityFromCertificate returns (nil, nil) when the extension is + // absent, an error when there is more than one or when its contents are + // malformed, empty, or carry a purpose other than atunnel. + identity, err := substratex509.ActorIdentityFromCertificate(leaf) + if err != nil { + return nil, fmt.Errorf("actor certificate has no single valid ActorIdentity extension: %w", err) + } + if identity == nil { + return nil, fmt.Errorf("actor certificate has no ActorIdentity extension") + } + // Restate what ActorIdentityFromCertificate enforces. The gateway is the + // component that gets hurt if that helper ever loosens, and "reject anything + // not scoped to atunnel" is the property this endpoint depends on: a + // certificate minted for some future purpose must not open a tunnel. + if identity.Atespace == "" || identity.ActorName == "" || identity.ActorUid == "" { + return nil, fmt.Errorf("actor certificate identity is incomplete") + } + if identity.Purpose != substratex509.ActorIdentityPurposeAtunnel { + return nil, fmt.Errorf("actor certificate purpose %q is not %q", + identity.Purpose, substratex509.ActorIdentityPurposeAtunnel) + } + return identity, nil +} + +// parseXFCCChain extracts the presented certificate chain, leaf first, from an +// x-forwarded-client-cert header value. +func parseXFCCChain(header string) ([]*x509.Certificate, error) { + // One element per proxy hop. SANITIZE_SET makes Envoy the only writer, so + // anything but exactly one element means either an unexpected proxy in front + // of the gateway or a listener that lost SANITIZE_SET — in both cases we no + // longer know which element describes our actual peer, so refuse to guess. + elements := splitXFCCUnquoted(header, ',') + if len(elements) != 1 { + return nil, fmt.Errorf("expected exactly one %s element, got %d", forwardedClientCertHeader, len(elements)) + } + encoded, ok := xfccValue(elements[0], xfccChainKey) + if !ok { + return nil, fmt.Errorf("%s carries no %q value", forwardedClientCertHeader, xfccChainKey) + } + // Envoy percent-encodes the PEM. PathUnescape, not QueryUnescape: base64 + // bodies contain '+', and query unescaping would decode it to a space and + // silently corrupt the DER. + chainPEM, err := url.PathUnescape(encoded) + if err != nil { + return nil, fmt.Errorf("decoding the client certificate chain: %w", err) + } + + var chain []*x509.Certificate + rest := []byte(chainPEM) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parsing the client certificate chain: %w", err) + } + chain = append(chain, cert) + } + if len(chain) == 0 { + return nil, fmt.Errorf("%s carries no certificate", forwardedClientCertHeader) + } + return chain, nil +} + +// xfccValue returns the value of key in one x-forwarded-client-cert element. +// Keys are matched case-insensitively; Envoy emits "Chain", but the header is +// consumed by enough different proxies that assuming its casing is not worth +// the failure mode. +func xfccValue(element, key string) (string, bool) { + for _, pair := range splitXFCCUnquoted(element, ';') { + k, v, found := strings.Cut(pair, "=") + if !found || !strings.EqualFold(strings.TrimSpace(k), key) { + continue + } + return unquoteXFCC(strings.TrimSpace(v)), true + } + return "", false +} + +// splitXFCCUnquoted splits on sep, ignoring separators inside a quoted value. +// x-forwarded-client-cert quotes any value containing its own delimiters, which +// the PEM ones always do. +func splitXFCCUnquoted(s string, sep rune) []string { + var parts []string + var current strings.Builder + quoted := false + escaped := false + for _, r := range s { + switch { + case escaped: + current.WriteRune(r) + escaped = false + case quoted && r == '\\': + current.WriteRune(r) + escaped = true + case r == '"': + quoted = !quoted + current.WriteRune(r) + case r == sep && !quoted: + parts = append(parts, current.String()) + current.Reset() + default: + current.WriteRune(r) + } + } + parts = append(parts, current.String()) + + trimmed := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + trimmed = append(trimmed, part) + } + } + return trimmed +} + +// unquoteXFCC strips the surrounding quotes from an x-forwarded-client-cert +// value and undoes the backslash escaping inside them. +func unquoteXFCC(value string) string { + if len(value) < 2 || !strings.HasPrefix(value, `"`) || !strings.HasSuffix(value, `"`) { + return value + } + inner := value[1 : len(value)-1] + var out strings.Builder + escaped := false + for _, r := range inner { + if escaped { + out.WriteRune(r) + escaped = false + continue + } + if r == '\\' { + escaped = true + continue + } + out.WriteRune(r) + } + return out.String() +} + +// mapEgressIdentityError converts a GetActor failure into a client-facing +// ext_proc denial. An unknown actor is treated as a forbidden (the actor was +// deleted out from under a still-valid certificate); transient control-plane +// failures fail closed with 503. +func mapEgressIdentityError(atespace, actorName string, err error) error { + switch status.Code(err) { + case codes.NotFound: + return newReqError(envoy_type.StatusCode_Forbidden, + "egress denied: unknown actor %q/%q", atespace, actorName) + case codes.Unavailable, codes.DeadlineExceeded: + return newReqError(envoy_type.StatusCode_ServiceUnavailable, + "egress identity check unavailable for %q/%q: %v", atespace, actorName, err) + default: + return newReqError(envoy_type.StatusCode_Forbidden, + "egress denied for %q/%q: %v", atespace, actorName, err) + } +} + +// loadActorIdentityRoots reads the actor-identity CA trust bundle the egress +// gateway verifies actor client certificates against. +func loadActorIdentityRoots(pemBytes []byte) (*x509.CertPool, error) { + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(pemBytes) { + return nil, fmt.Errorf("actor-identity CA bundle contains no certificates") + } + return roots, nil +} diff --git a/cmd/atenet/internal/router/extproc_egress_test.go b/cmd/atenet/internal/router/extproc_egress_test.go new file mode 100644 index 000000000..48ab0c0e9 --- /dev/null +++ b/cmd/atenet/internal/router/extproc_egress_test.go @@ -0,0 +1,698 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/url" + "strings" + "testing" + "time" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/agent-substrate/substrate/internal/substratex509" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// connectRequest builds a RequestHeaders ProcessingRequest for an egress +// CONNECT, optionally attributed to a listener. filterKey is the ext_proc +// filter name Envoy keys the attributes map by. +func connectRequest(filterKey, listener string) *extprocv3.ProcessingRequest { + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{ + Headers: []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("CONNECT")}, + {Key: ":authority", RawValue: []byte("10.0.0.9:443")}, + }, + }, + }, + }, + } + if listener != "" { + req.Attributes = map[string]*structpb.Struct{ + filterKey: { + Fields: map[string]*structpb.Value{ + ListenerNameAttribute: structpb.NewStringValue(listener), + }, + }, + } + } + return req +} + +func TestIsEgressRequest(t *testing.T) { + tests := []struct { + name string + filterKey string + listener string + want bool + }{ + { + name: "egress listener", + filterKey: "envoy.filters.http.ext_proc", + listener: EgressListenerName, + want: true, + }, + { + name: "egress listener under a renamed filter", + filterKey: "some.custom.ext_proc.name", + listener: EgressListenerName, + want: true, + }, + { + // The pre-listener-dispatch hole: an external client sending + // CONNECT to the ingress gateway must not reach the egress handler, + // whose denials would otherwise report whether an arbitrary actor + // exists and is running. + name: "CONNECT on the ingress HTTP listener", + filterKey: "envoy.filters.http.ext_proc", + listener: IngressHTTPListener, + want: false, + }, + { + name: "CONNECT on the ingress HTTPS listener", + filterKey: "envoy.filters.http.ext_proc", + listener: IngressHTTPSListener, + want: false, + }, + { + // A listener that never requested the attribute falls back to + // ingress, the fail-safe direction. + name: "no attributes at all", + listener: "", + want: false, + }, + { + name: "unrecognised listener", + filterKey: "envoy.filters.http.ext_proc", + listener: "some-other-listener", + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isEgressRequest(connectRequest(tc.filterKey, tc.listener)); got != tc.want { + t.Errorf("isEgressRequest() = %v, want %v", got, tc.want) + } + }) + } +} + +// A request the client dresses up to look like egress must not be enough: only +// the Envoy-asserted listener name selects the egress handler. +func TestIsEgressRequestIgnoresClientSuppliedAttributeHeader(t *testing.T) { + req := connectRequest("envoy.filters.http.ext_proc", IngressHTTPListener) + rh := req.GetRequestHeaders().GetHeaders() + rh.Headers = append(rh.Headers, + &corev3.HeaderValue{Key: ListenerNameAttribute, RawValue: []byte(EgressListenerName)}, + &corev3.HeaderValue{Key: "x-envoy-listener-name", RawValue: []byte(EgressListenerName)}, + ) + + if isEgressRequest(req) { + t.Error("isEgressRequest() = true for a client-forged listener header, want false") + } +} + +// --- actor certificate authentication ------------------------------------- + +const ( + testEgressAtespace = "default" + testEgressActor = "my-actor" + testEgressActorUID = "1b4e28ba-2fa1-11d2-883f-0016d3cca427" +) + +// testCA is a throwaway CA standing in for the actor-identity CA. +type testCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey +} + +func newTestCA(t *testing.T, commonName string) *testCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating CA key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: commonName}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatalf("creating CA certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parsing CA certificate: %v", err) + } + return &testCA{cert: cert, key: key} +} + +func (ca *testCA) roots() *x509.CertPool { + pool := x509.NewCertPool() + pool.AddCert(ca.cert) + return pool +} + +// oidActorIdentity mirrors the unexported OID substratex509 encodes the +// ActorIdentity extension under: the Substrate PEN arc, sub-identifier 2. +var oidActorIdentity = append(append(asn1.ObjectIdentifier{}, substratex509.GoogleSubstratePEN...), 2) + +// addActorIdentityUnchecked encodes identity into template the way +// substratex509.AddActorIdentityToCertificate does, minus its validation, so +// tests can mint the malformed identities a real CA would refuse to produce and +// confirm the gateway rejects them anyway. +func addActorIdentityUnchecked(identity *substratex509.ActorIdentity, template *x509.Certificate) error { + value, err := json.Marshal(identity) + if err != nil { + return err + } + template.ExtraExtensions = append(template.ExtraExtensions, pkix.Extension{Id: oidActorIdentity, Value: value}) + return nil +} + +// actorCertOptions mutates the leaf template so each test can break exactly one +// property of an otherwise-valid actor certificate. +type actorCertOptions struct { + identity *substratex509.ActorIdentity + extraIdentity *substratex509.ActorIdentity + mutate func(*x509.Certificate) +} + +// issueActorCert mints a leaf off ca, mirroring what ateapi's actoridentity +// service produces. +func (ca *testCA) issueActorCert(t *testing.T, opts actorCertOptions) *x509.Certificate { + t.Helper() + cert, err := x509.ParseCertificate(ca.issueActorCertDER(t, opts)) + if err != nil { + t.Fatalf("parsing leaf certificate: %v", err) + } + return cert +} + +// issueActorCertDER is issueActorCert without the parse, for the certificates +// crypto/x509 itself refuses to read back. +func (ca *testCA) issueActorCertDER(t *testing.T, opts actorCertOptions) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generating leaf key: %v", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: testEgressActor}, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + IsCA: false, + } + identity := opts.identity + if identity == nil { + identity = &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + } + } + // AddActorIdentityToCertificate validates its input, so identities a real CA + // would refuse to mint are encoded directly. + if err := addActorIdentityUnchecked(identity, template); err != nil { + t.Fatalf("adding ActorIdentity extension: %v", err) + } + if opts.extraIdentity != nil { + if err := addActorIdentityUnchecked(opts.extraIdentity, template); err != nil { + t.Fatalf("adding second ActorIdentity extension: %v", err) + } + } + if opts.mutate != nil { + opts.mutate(template) + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatalf("signing leaf certificate: %v", err) + } + return der +} + +// xfccHeader renders chain the way Envoy's SANITIZE_SET + +// set_current_client_cert_details{chain: true} does. +func xfccHeader(chain ...*x509.Certificate) string { + der := make([][]byte, 0, len(chain)) + for _, cert := range chain { + der = append(der, cert.Raw) + } + return xfccHeaderDER(der...) +} + +func xfccHeaderDER(chain ...[]byte) string { + var buf strings.Builder + for _, der := range chain { + _ = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}) + } + return fmt.Sprintf(`By=spiffe://cluster.local/ns/ate-system/sa/ateway-egress;Hash=abc123;Chain="%s"`, + url.PathEscape(buf.String())) +} + +// egressServer builds an ExtProcServer whose GetActor returns actor/err. +func egressServer(roots *x509.CertPool, actor *ateapipb.Actor, err error) *ExtProcServer { + client := &egressMockClient{actor: actor, err: err} + return NewExtProcServer(50051, client, nil, ParkedRequestConfig{}, nil, false, roots) +} + +type egressMockClient struct { + ateapipb.ControlClient + actor *ateapipb.Actor + err error +} + +func (m *egressMockClient) GetActor(context.Context, *ateapipb.GetActorRequest, ...grpc.CallOption) (*ateapipb.Actor, error) { + if m.err != nil { + return nil, m.err + } + return m.actor, nil +} + +func runningActor() *ateapipb.Actor { + return &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testEgressAtespace, + Name: testEgressActor, + Uid: testEgressActorUID, + }, + Status: ateapipb.Actor_STATUS_RUNNING, + } +} + +// egressHeaders builds the CONNECT the egress listener hands to ext_proc. +func egressHeaders(xfcc string) *extprocv3.HttpHeaders { + headers := []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("CONNECT")}, + {Key: ":authority", RawValue: []byte("93.184.216.34:80")}, + } + if xfcc != "" { + headers = append(headers, &corev3.HeaderValue{Key: forwardedClientCertHeader, RawValue: []byte(xfcc)}) + } + return &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{Headers: headers}} +} + +func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) { + t.Helper() + if err == nil { + t.Fatalf("expected a denial with status %d, got none", want) + } + var re *reqError + if !errors.As(err, &re) { + t.Fatalf("error %v is not a *reqError", err) + } + if re.statusCode != int(want) { + t.Errorf("status = %d, want %d (%v)", re.statusCode, want, err) + } +} + +func TestHandleEgressRequestHeadersAllowsVerifiedActor(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + s := egressServer(ca.roots(), runningActor(), nil) + + resp, _, _, _, _, resume, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf))) + if err != nil { + t.Fatalf("handleEgressRequestHeaders() error = %v, want nil", err) + } + if resp == nil { + t.Fatal("handleEgressRequestHeaders() returned no response") + } + if resume != ResumeOutcomeNone { + t.Errorf("resume outcome = %q, want %q", resume, ResumeOutcomeNone) + } +} + +// Every way an actor certificate can fail to prove an identity has to end in a +// denial, never in a tunnel. +func TestHandleEgressRequestHeadersRejectsBadCertificates(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + otherCA := newTestCA(t, "some-other-ca") + + tests := []struct { + name string + xfcc func(t *testing.T) string + want envoy_type.StatusCode + }{ + { + name: "no client certificate at all", + xfcc: func(*testing.T) string { return "" }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "signed by an unknown CA", + xfcc: func(t *testing.T) string { + return xfccHeader(otherCA.issueActorCert(t, actorCertOptions{})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "expired", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.NotBefore = time.Now().Add(-2 * time.Hour) + c.NotAfter = time.Now().Add(-time.Hour) + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "not yet valid", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.NotBefore = time.Now().Add(time.Hour) + c.NotAfter = time.Now().Add(2 * time.Hour) + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "no ClientAuth EKU", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + // An empty EKU means "any usage" to crypto/x509 and would pass + // VerifyOptions.KeyUsages; the explicit ClientAuth check is what + // catches it. + name: "empty EKU", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.ExtKeyUsage = nil + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "is a CA certificate", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.IsCA = true + c.KeyUsage |= x509.KeyUsageCertSign + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "no ActorIdentity extension", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{mutate: func(c *x509.Certificate) { + c.ExtraExtensions = nil + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + // Stays in DER: crypto/x509 refuses to parse a certificate with a + // duplicate extension OID at all, so the helper cannot hand back an + // *x509.Certificate here. That refusal is the first of the two + // layers guarding "exactly one ActorIdentity" — this case proves the + // handler denies rather than panics when the parse fails, and + // substratex509 rejects a second copy if a parser ever allowed one. + name: "two ActorIdentity extensions", + xfcc: func(t *testing.T) string { + return xfccHeaderDER(ca.issueActorCertDER(t, actorCertOptions{ + extraIdentity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: "a-different-actor", + ActorUid: "8f14e45f-ceea-467a-9575-25a0d5d5e4b0", + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }, + })) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "generic purpose", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + Purpose: "generic", + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "missing purpose", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "empty atespace", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + ActorName: testEgressActor, + ActorUid: testEgressActorUID, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "empty actor name", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorUid: testEgressActorUID, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "empty actor UID", + xfcc: func(t *testing.T) string { + return xfccHeader(ca.issueActorCert(t, actorCertOptions{identity: &substratex509.ActorIdentity{ + Atespace: testEgressAtespace, + ActorName: testEgressActor, + Purpose: substratex509.ActorIdentityPurposeAtunnel, + }})) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + // SANITIZE_SET makes Envoy the only writer of the header, so two + // elements means we can no longer tell which one is our peer. + name: "two XFCC elements", + xfcc: func(t *testing.T) string { + leaf := ca.issueActorCert(t, actorCertOptions{}) + return xfccHeader(leaf) + "," + xfccHeader(leaf) + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "XFCC without a Chain value", + xfcc: func(*testing.T) string { + return `By=spiffe://cluster.local/ns/ate-system/sa/ateway-egress;Hash=abc123` + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "XFCC Chain that is not a certificate", + xfcc: func(*testing.T) string { + return `Chain="` + url.PathEscape("-----BEGIN CERTIFICATE-----\nbm90LWEtY2VydA==\n-----END CERTIFICATE-----\n") + `"` + }, + want: envoy_type.StatusCode_Forbidden, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := egressServer(ca.roots(), runningActor(), nil) + _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(tc.xfcc(t))) + wantStatus(t, err, tc.want) + }) + } +} + +// The certificate authenticates; these cover what the control plane says about +// the actor it names. +func TestHandleEgressRequestHeadersAuthorization(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + + tests := []struct { + name string + actor *ateapipb.Actor + err error + want envoy_type.StatusCode + }{ + { + // The actor was deleted and recreated under the same name: the + // certificate is still cryptographically valid but names a UID that + // no longer exists, and must not carry over to the successor. + name: "actor UID does not match the certificate", + actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testEgressAtespace, + Name: testEgressActor, + Uid: "d41d8cd9-8f00-4204-a980-0998ecf8427e", + }, + Status: ateapipb.Actor_STATUS_RUNNING, + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "actor has no UID", + actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testEgressAtespace, Name: testEgressActor}, + Status: ateapipb.Actor_STATUS_RUNNING, + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "actor is not running", + actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: testEgressAtespace, + Name: testEgressActor, + Uid: testEgressActorUID, + }, + Status: ateapipb.Actor_STATUS_SUSPENDED, + }, + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "actor no longer exists", + err: status.Error(codes.NotFound, "no such actor"), + want: envoy_type.StatusCode_Forbidden, + }, + { + name: "control plane unreachable", + err: status.Error(codes.Unavailable, "ateapi is down"), + want: envoy_type.StatusCode_ServiceUnavailable, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := egressServer(ca.roots(), tc.actor, tc.err) + leaf := ca.issueActorCert(t, actorCertOptions{}) + _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf))) + wantStatus(t, err, tc.want) + }) + } +} + +// An ingress-only router has no actor-identity CA. If an egress CONNECT somehow +// reaches it, it must fail closed rather than tunnel unauthenticated traffic. +func TestHandleEgressRequestHeadersWithoutConfiguredCA(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + s := egressServer(nil, runningActor(), nil) + + _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf))) + wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) +} + +func TestHandleEgressRequestHeadersRejectsNonConnect(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + s := egressServer(ca.roots(), runningActor(), nil) + + headers := egressHeaders(xfccHeader(leaf)) + for _, h := range headers.Headers.Headers { + if h.Key == ":method" { + h.RawValue = []byte("GET") + } + } + _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), headers) + wantStatus(t, err, envoy_type.StatusCode_MethodNotAllowed) +} + +// PEM bodies routinely contain '+'. Decoding the header as a query string would +// turn those into spaces and corrupt the DER, so pin the round trip. +func TestParseXFCCChainPreservesPlusInPEM(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + // Serials differ per certificate, so mint until one encodes with a '+'. + var leaf *x509.Certificate + for i := 0; i < 50; i++ { + candidate := ca.issueActorCert(t, actorCertOptions{}) + if strings.Contains(string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: candidate.Raw})), "+") { + leaf = candidate + break + } + } + if leaf == nil { + t.Skip("no certificate with a '+' in its PEM body after 50 attempts") + } + + chain, err := parseXFCCChain(xfccHeader(leaf)) + if err != nil { + t.Fatalf("parseXFCCChain() error = %v", err) + } + if len(chain) != 1 || !chain[0].Equal(leaf) { + t.Fatalf("parseXFCCChain() did not round-trip the certificate") + } +} + +func TestParseXFCCChainIncludesIntermediates(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + + chain, err := parseXFCCChain(xfccHeader(leaf, ca.cert)) + if err != nil { + t.Fatalf("parseXFCCChain() error = %v", err) + } + if len(chain) != 2 { + t.Fatalf("parseXFCCChain() returned %d certificates, want 2", len(chain)) + } + if !chain[0].Equal(leaf) { + t.Error("parseXFCCChain() did not return the leaf first") + } +} diff --git a/cmd/atenet/internal/router/extproc_in.go b/cmd/atenet/internal/router/extproc_in.go index 6ab2d2a6d..ce974983f 100644 --- a/cmd/atenet/internal/router/extproc_in.go +++ b/cmd/atenet/internal/router/extproc_in.go @@ -28,12 +28,14 @@ type requestMetadata struct { headers map[string]string path string host string + method string } func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata { headersMap := make(map[string]string) var path string var host string + var method string for _, h := range headers { k := strings.ToLower(h.Key) @@ -49,12 +51,16 @@ func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata { if k == authorityHeader || k == "host" { host = val } + if k == ":method" { + method = val + } } return &requestMetadata{ headers: headersMap, path: path, host: host, + method: method, } } diff --git a/cmd/atenet/internal/router/extproc_test.go b/cmd/atenet/internal/router/extproc_test.go index 5173b40db..cf3901521 100644 --- a/cmd/atenet/internal/router/extproc_test.go +++ b/cmd/atenet/internal/router/extproc_test.go @@ -59,7 +59,7 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}}, nil }, - }, nil, ParkedRequestConfig{}, nil, false) + }, nil, ParkedRequestConfig{}, nil, false, nil) reqHeaders := &extprocv3.HttpHeaders{ Headers: &corev3.HeaderMap{ @@ -196,7 +196,7 @@ func TestExtProcHeadersEvaluation(t *testing.T) { // Parking disabled: these cases assert fail-fast mapping of resume // errors (e.g. FailedPrecondition -> immediate 503). Parking behavior // is covered separately in TestExtProc_ParkingLotFull and resumer_test.go. - s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{}, nil, false) + s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{}, nil, false, nil) reqHeaders := &extprocv3.HttpHeaders{ Headers: &corev3.HeaderMap{ @@ -277,7 +277,7 @@ func TestExtProc_ParkingLotFull(t *testing.T) { // A 1-slot lot with the slot already occupied deterministically simulates a // full lot without needing a concurrent in-flight request. - s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false) + s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false, nil) release, ok := s.parking.enter(context.Background()) if !ok { t.Fatal("priming enter should be admitted") @@ -396,7 +396,7 @@ func TestRecordRouteDuration_Attributes(t *testing.T) { t.Fatalf("failed to create histogram: %v", err) } - s := NewExtProcServer(50051, nil, h, ParkedRequestConfig{}, nil, false) + s := NewExtProcServer(50051, nil, h, ParkedRequestConfig{}, nil, false, nil) s.recordRouteDuration(context.Background(), 10*time.Millisecond, "team-a-ns", "tmpl-a", classifyOutcome(nil), string(ResumeOutcomeTriggered)) var rm metricdata.ResourceMetrics diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 49fd3e503..5a0298d64 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -16,6 +16,7 @@ package router import ( "context" + "crypto/x509" "errors" "fmt" "log/slog" @@ -215,7 +216,21 @@ func (s *RouterServer) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to create parking metrics: %w", err) } - s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority()) + // Load the actor-identity CA up front so a missing or unusable bundle + // fails startup, rather than turning into a 503 on the first actor + // egress attempt. An unset flag is the ingress-only case and is fine. + var actorIdentityRoots *x509.CertPool + if s.cfg.ActorIdentityCAFile != "" { + pemBytes, err := os.ReadFile(s.cfg.ActorIdentityCAFile) + if err != nil { + return fmt.Errorf("reading --actor-identity-ca-file: %w", err) + } + actorIdentityRoots, err = loadActorIdentityRoots(pemBytes) + if err != nil { + return fmt.Errorf("loading --actor-identity-ca-file %q: %w", s.cfg.ActorIdentityCAFile, err) + } + } + s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority(), actorIdentityRoots) } s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg) diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index 2f6e97cbd..7f5234097 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -79,7 +79,7 @@ func TestStatuszEndpoint(t *testing.T) { t.Fatalf("Failed generating router server: %v", err) } - srv.extprocSrv = NewExtProcServer(cfg.ExtprocPort, &mockClient{}, nil, defaultParkedRequestConfig(), nil, false) + srv.extprocSrv = NewExtProcServer(cfg.ExtprocPort, &mockClient{}, nil, defaultParkedRequestConfig(), nil, false, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 5238e0a6b..7a6c1471c 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -79,6 +79,7 @@ function usage() { echo "" echo " --create-jwt-authority-pool-secret Create JWT authority pool secret" echo " --create-actor-id-ca-pool-secret Create actor ID CA pool secret" + echo " --create-actor-id-ca-certs-secret Create actor ID CA certs secret" echo " --create-podcertificate-controller-cas Create podcertificate controller CAs" echo " --create-valkey-ca-certs-secret Create Valkey CA certs secret" echo " --create-api-server-env-vars Create ate-api-server env vars" @@ -222,10 +223,13 @@ apply_otel_config() { } # Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. +# The namespace defaults to the podcertificate controller's, where the signer +# CAs live; the actor-identity CA pool is in ate-system, so it passes its own. ca_pool_root_pem() { local secret="$1" + local namespace="${2:-podcertificate-controller-system}" local pool_json="" - pool_json=$(run_kubectl get secret -n podcertificate-controller-system "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) + pool_json=$(run_kubectl get secret -n "${namespace}" "${secret}" -o jsonpath='{.data.pool}' | base64 --decode) local der_base64="" der_base64=$(echo "${pool_json}" | grep -o '"RootCertificateDER":"[^"]*' | sed 's/"RootCertificateDER":"//') echo "${der_base64}" | base64 --decode | openssl x509 -inform der -outform pem @@ -275,6 +279,43 @@ create_actor_id_ca_pool_secret() { --secret-namespace=ate-system } +# --------------------------------------------------------------------------- +# SEE(lior): actor-identity CA trust bundle for the egress PEP. +# +# The egress gateway has to verify actor client certificates, which means it +# needs the actor-identity CA *root* — and only the root. The authoritative +# copy today is the actor-id-ca-pool Secret, but its pool.json also carries the +# CA signing key, so mounting that Secret into the gateway would put the key +# that mints every actor identity inside a pod that only ever needs to verify +# them. This derives a cert-only Secret instead, following exactly the pattern +# create_valkey_ca_certs_secret already uses for the signer roots. +# +# TODO(liorlieberman): revisit. The other CAs reach their consumers as +# ClusterTrustBundles published by the podcertificate controller, and the +# actor-identity CA arguably should too — then the gateway would project a +# trust bundle like it already does for servicedns/podidentity and this +# install-time Secret would go away. Doing that needs a signer/controller path +# that does not exist yet, so this is the interim shape. +# --------------------------------------------------------------------------- +create_actor_id_ca_certs_secret() { + log_step "create_actor_id_ca_certs_secret" + # Extract into its own variable first: errexit cannot see a substitution fail + # inside the create-secret argument list, which would silently produce an + # empty trust bundle and an egress gateway that rejects every actor. + local actorid_root="" + actorid_root=$(ca_pool_root_pem actor-id-ca-pool ate-system) + if [[ -z "${actorid_root}" ]]; then + echo "error: failed to extract the actor-identity CA root for actor-id-ca-certs" >&2 + return 1 + fi + + run_kubectl create secret generic actor-id-ca-certs \ + --from-literal=ca.crt="${actorid_root}" \ + -n ate-system \ + --dry-run=client -o yaml \ + | run_kubectl apply -f - +} + create_podcertificate_controller_cas() { log_step "create_podcertificate_controller_cas" run_kubectl create namespace podcertificate-controller-system || true @@ -405,6 +446,7 @@ deploy_ate_system() { run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout=120s run_kubectl rollout status deployment/ate-controller -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s + run_kubectl rollout status deployment/ateway-egress -n ate-system --timeout=120s run_kubectl rollout status statefulset/valkey-cluster -n ate-system --timeout=120s run_kubectl rollout status daemonset/atelet -n ate-system --timeout=120s } @@ -416,6 +458,9 @@ ensure_apiserver_prerequisites() { || create_jwt_authority_pool_secret run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ || create_actor_id_ca_pool_secret + # SEE(lior): derived from actor-id-ca-pool above, so it must come after it. + run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ + || create_actor_id_ca_certs_secret run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ || create_podcertificate_controller_cas run_kubectl get secret -n ate-system valkey-ca-certs >/dev/null 2>&1 \ @@ -476,11 +521,15 @@ deploy_atenet() { router_manifest="$(render_atenet_router_manifest)" echo "${router_manifest}" | run_kubectl apply -f - + run_ko apply -f manifests/ate-install/ateway-egress.yaml run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s - # The Deployment in atenet-dns.yaml is named "dns"; every other resource in - # that file is "atenet-dns". Waiting on the filename rather than the actual - # Deployment made this step fail with NotFound on every successful deploy. + run_kubectl rollout status deployment/ateway-egress -n ate-system --timeout=120s + # SEE(lior): this branch also added a `deployment/atenet-dns` wait here, which + # main has since fixed to `deployment/dns` (the Deployment in atenet-dns.yaml + # is named "dns"; every other resource in that file is "atenet-dns", so the + # old name was NotFound on every successful deploy). Dropped this branch's + # duplicate in favour of main's corrected line. run_kubectl rollout status deployment/dns -n ate-system --timeout=120s } @@ -599,6 +648,8 @@ delete_atenet() { run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-router.yaml run_kubectl delete --ignore-not-found \ -f manifests/ate-install/components/agentgateway/configmap.yaml + run_kubectl delete --ignore-not-found -f manifests/ate-install/ateway-egress.yaml + run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-dns.yaml } deploy_benchmarks() { @@ -734,6 +785,7 @@ while [[ "$#" -gt 0 ]]; do --create-jwt-authority-pool-secret) create_jwt_authority_pool_secret ;; --create-actor-id-ca-pool-secret) create_actor_id_ca_pool_secret ;; + --create-actor-id-ca-certs-secret) create_actor_id_ca_certs_secret ;; --create-podcertificate-controller-cas) create_podcertificate_controller_cas ;; --create-valkey-ca-certs-secret) create_valkey_ca_certs_secret ;; --create-api-server-env-vars) create_api_server_env_vars ;; diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 560f32e20..47d7e6238 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -104,6 +104,15 @@ spec: - --actor-id-ca-pool=/run/actor-id-ca-pool/pool.json - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + # ONLY UNTIL THE EGRESS API IS DECIDED, FOR POC: turn on pluggable actor + # egress cluster-wide. ateapi stamps this address onto every atelet + # Run/Restore, ateom hands it to atunnel, and actor TCP egress is + # transparently redirected (nftables) into atunnel, which wraps it in + # mTLS + HTTP CONNECT to the egress gateway. + # + # SEE(lior): this flag lived on atelet on the pre-rebase branch; PR 708 + # moved egress-gateway configuration to ateapi, so it is set here now. + - --egress-gateway-address=ateway-egress.ate-system.svc:443 # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s - --drain-timeout=15s diff --git a/manifests/ate-install/ateway-egress.yaml b/manifests/ate-install/ateway-egress.yaml new file mode 100644 index 000000000..37d7f9d7a --- /dev/null +++ b/manifests/ate-install/ateway-egress.yaml @@ -0,0 +1,393 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# contract expected by atunnel's egress client: +# * downstream mTLS on :443 (present servicedns identity, require + verify an +# actor-identity client cert), +# * terminate the actor's HTTP CONNECT and tunnel raw TCP to the CONNECT +# authority (the actor's original destination, always sent as IP:port). +# +# SEE(lior): the client cert is the actor's own identity — minted per actor by +# ateapi's actoridentity service off the actor-identity CA and carrying the +# ActorIdentity X.509 extension — not the pod's podidentity cert. The pod cert +# says "some substrate pod"; only the actor cert says *which actor*, which is +# the thing the gateway has to authorize. The CONNECT carries no identity +# headers at all; everything the PEP decides on comes out of the certificate. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ateway-egress + namespace: ate-system +--- +# RBAC for the co-located ext_proc sidecar (atenet router), which watches +# ActorTemplates as part of its normal operation. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ateway-egress +rules: +- apiGroups: + - "ate.dev" + resources: + - actortemplates + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ateway-egress +subjects: +- kind: ServiceAccount + name: ateway-egress + namespace: ate-system +roleRef: + kind: ClusterRole + name: ateway-egress + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ateway-egress + namespace: ate-system +data: + envoy.yaml: | + admin: + address: + socket_address: { address: 0.0.0.0, port_value: 15000 } + static_resources: + listeners: + - name: egress + address: + socket_address: { address: 0.0.0.0, port_value: 443 } + filter_chains: + - transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + require_client_certificate: true + common_tls_context: + tls_certificates: + # Gateway server identity (servicedns signer). credential-bundle.pem + # holds the leaf cert and key concatenated. watched_directory picks + # up kubelet's projected certificate rotation without a restart. + - certificate_chain: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } + private_key: { filename: /run/servicedns.podcert.ate.dev/credential-bundle.pem } + watched_directory: { path: /run/servicedns.podcert.ate.dev } + validation_context: + # Actors authenticate with an actor-identity client cert. + # Envoy enforces chain, signature, and validity period here, so + # an unsigned or expired actor cert never reaches ext_proc; the + # ActorIdentity extension is checked there because Envoy cannot + # read custom X.509 extensions. + trusted_ca: { filename: /run/actor-id-ca-certs/ca.crt } + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: egress_connect + codec_type: HTTP1 + upgrade_configs: + - upgrade_type: CONNECT + # SEE(lior): this pair is how the actor's certificate reaches the + # ext_proc handler. ext_proc can request Envoy attributes, but none + # of them carry a custom X.509 extension, so the only way to check + # ActorIdentity in Go is to have Envoy hand over the raw chain. + # SANITIZE_SET drops whatever x-forwarded-client-cert the client + # sent and writes Envoy's own view of the verified peer, and + # chain: true puts the full URL-encoded PEM chain in it. Removing + # either of these breaks egress closed: the handler sees no XFCC + # and denies every CONNECT. + forward_client_cert_details: SANITIZE_SET + set_current_client_cert_details: + chain: true + # Emit the access log as soon as the CONNECT tunnel is established + # (atunnel keeps the tunnel open, so the default log-on-close would + # not fire during the demo). + access_log_options: + flush_log_on_tunnel_successfully_established: true + access_log: + - name: envoy.access_loggers.stdout + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + text_format_source: + # SEE(lior): the peer fields come from the verified client + # certificate, not from request headers. The actor sends no + # identity headers any more, and logging attacker-controlled + # ones would have made this log lie about who egressed. + # peer= is the leaf subject (CN carries the actor); the + # atespace/UID live in the ActorIdentity extension, which + # Envoy cannot format — the ext_proc handler logs those. + inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer=%DOWNSTREAM_PEER_SUBJECT% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" + route_config: + name: connect_route + virtual_hosts: + - name: connect + domains: ["*"] + routes: + - match: { connect_matcher: {} } + route: + cluster: egress_forward_proxy + upgrade_configs: + - upgrade_type: CONNECT + connect_config: {} + http_filters: + # Actor-identity authorization: on every egress CONNECT, ext_proc + # reads the actor certificate out of x-forwarded-client-cert, + # re-verifies it against the actor-identity CA, pulls the + # ActorIdentity extension, and asks the ate API whether that UID is + # a real, running actor. Denials come back as an immediate 403. + # Fails closed if the router is down. + - name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + grpc_service: + envoy_grpc: + cluster_name: ext_proc_server + timeout: 2s + failure_mode_allow: false + # How the ext_proc server tells egress from ingress. It applies + # opposite trust models to the two directions, and dispatches on + # this Envoy-asserted listener name so that no client can select + # the egress path by crafting a request. The value must match + # EgressListenerName in cmd/atenet/internal/router/extproc_egress.go; + # renaming the listener above without updating it fails closed + # (egress requests take the ingress path and 404). + request_attributes: + - xds.listener_name + processing_mode: + request_header_mode: SEND + response_header_mode: SKIP + request_body_mode: NONE + response_body_mode: NONE + request_trailer_mode: SKIP + response_trailer_mode: SKIP + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + # ext_proc gRPC server = the atenet router, co-located in this pod as a + # sidecar and called over localhost (same topology the ingress gateway uses + # for its Envoy + ext_proc). + - name: ext_proc_server + type: STATIC + lb_policy: ROUND_ROBIN + connect_timeout: 1s + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: ext_proc_server + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 50051 + # Dials the terminated-CONNECT target (IP:port from the authority). atunnel + # always sends an IP:port, so DNS resolution is effectively a passthrough. + - name: egress_forward_proxy + lb_policy: CLUSTER_PROVIDED + connect_timeout: 5s + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: egress_dns_cache + dns_lookup_family: V4_ONLY +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ateway-egress + namespace: ate-system + labels: + app: ateway-egress +spec: + replicas: 1 + selector: + matchLabels: + app: ateway-egress + template: + metadata: + labels: + app: ateway-egress + spec: + serviceAccountName: ateway-egress + securityContext: + # Allow the non-root envoy user to bind :443. + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + containers: + - name: envoy + image: envoyproxy/envoy:v1.34-latest + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 65532 + args: + - -c + - /etc/envoy/envoy.yaml + - --service-node + - ateway-egress + - --service-cluster + - ateway-egress + ports: + - name: https + containerPort: 443 + - name: admin + containerPort: 15000 + readinessProbe: + httpGet: + path: /ready + port: admin + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /ready + port: admin + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/envoy + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + # Co-located ext_proc server (the atenet router, ext_proc-only). The egress + # Envoy calls it over localhost to authenticate actor identity against the + # ate API on every CONNECT. This mirrors the ingress gateway topology + # (Envoy + ext_proc in one pod); a shared/standalone ext_proc is a future step. + - name: ext-proc + image: ko://github.com/agent-substrate/substrate/cmd/atenet + args: + - router + - --standalone + - --namespace=ate-system + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address=api.ate-system.svc:443 + - --ateapi-auth=mtls + # Same bundle Envoy validates the handshake against. The handler + # re-verifies the chain in Go and reads the ActorIdentity extension out + # of it; without this flag the router has no actor-identity roots and + # denies every egress CONNECT with a 503. + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + readinessProbe: + tcpSocket: + port: extproc + periodSeconds: 10 + volumeMounts: + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + volumes: + - name: config + configMap: + name: ateway-egress + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + # SEE(lior): the actor-identity CA root, as a cert-only Secret that + # install-ate.sh derives from the actor-id-ca-pool Secret. It is a plain + # Secret rather than a projected clusterTrustBundle — unlike the two above + # — because the actor-identity CA has no signer/controller publishing a + # trust bundle for it yet; its only in-cluster home is actor-id-ca-pool, + # whose pool.json also holds the CA *signing key*. Mounting that here would + # put the key that mints every actor identity into the pod that merely + # verifies them, so install-ate.sh extracts the root and nothing else. + # + # TODO(liorlieberman): revisit once the actor-identity CA is published as a + # ClusterTrustBundle; this volume then becomes a third projected source and + # the install-time Secret goes away. See create_actor_id_ca_certs_secret in + # hack/install-ate.sh. + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: ateway-egress + namespace: ate-system +spec: + type: ClusterIP + selector: + app: ateway-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 6a6040ac2..3e916e80f 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -23,6 +23,7 @@ resources: - ../ate-controller.yaml - ../atelet.yaml - ../atenet-dns.yaml + - ../ateway-egress.yaml - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 54a6b0b33..8be23e96f 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -26,6 +26,7 @@ resources: - ../ate-controller.yaml - ./atelet - ../atenet-dns.yaml + - ../ateway-egress.yaml - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml From 6b3e06927aa8cc8e6321c9ad0efdb38640829d79 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Mon, 27 Jul 2026 16:32:39 -0700 Subject: [PATCH 2/6] egress: add the egress demo and e2e coverage demos/egress is a small Actor that fetches a URL it is given and echoes the upstream status and body back, which makes the egress path observable from outside the sandbox. hack/install-demo-egress.sh registers it as a --deploy-demo-egress fixture and hack/verify-egress-demo.sh drives it and checks the atenet-egress logs for the corresponding authorized CONNECT. TestActorEgress in the networking suite covers the same path automatically: it creates an Actor from the demo template, POSTs a fetch request through atenet-router, and asserts 200. The suite's actor helper is parameterised by template so the ingress test keeps using the counter fixture. --- .github/workflows/pr-workflow.yaml | 4 + Makefile | 2 +- cmd/atenet/internal/router/extproc.go | 8 - cmd/atenet/internal/router/extproc_egress.go | 62 +++--- .../internal/router/extproc_egress_test.go | 46 ++-- demos/egress/README.md | 148 +++++++++++++ demos/egress/egress.yaml.tmpl | 56 +++++ demos/egress/main.go | 125 +++++++++++ demos/egress/main_test.go | 109 ++++++++++ demos/egress/test-egress.sh | 196 ++++++++++++++++++ hack/install-ate.sh | 14 +- hack/install-demo-egress.sh | 51 +++++ hack/verify-egress-demo.sh | 66 ++++++ internal/e2e/router_client.go | 21 +- internal/e2e/router_client_test.go | 78 +++++++ .../e2e/suites/networking/networking_test.go | 56 ++++- manifests/ate-install/ate-api-server.yaml | 2 +- ...{ateway-egress.yaml => atenet-egress.yaml} | 76 ++++--- manifests/ate-install/base/kustomization.yaml | 2 +- manifests/ate-install/kind/kustomization.yaml | 2 +- 20 files changed, 1017 insertions(+), 107 deletions(-) create mode 100644 demos/egress/README.md create mode 100644 demos/egress/egress.yaml.tmpl create mode 100644 demos/egress/main.go create mode 100644 demos/egress/main_test.go create mode 100755 demos/egress/test-egress.sh create mode 100644 hack/install-demo-egress.sh create mode 100644 hack/verify-egress-demo.sh create mode 100644 internal/e2e/router_client_test.go rename manifests/ate-install/{ateway-egress.yaml => atenet-egress.yaml} (86%) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 14d0c1681..e1441aaa6 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -96,6 +96,10 @@ jobs: run: hack/run-microvm-demo-kind.sh --ateapi-client-auth=${{ matrix.ateapi-client-auth }} - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + # TestActorEgress in the networking suite builds its Actor from the + # ate-demo-egress/egress ActorTemplate, so the fixture has to exist before. + run: hack/install-ate-kind.sh --deploy-demo-egress - name: Wait for micro-VM golden snapshot run: | kubectl --context kind-kind wait --for=condition=Ready \ diff --git a/Makefile b/Makefile index 3eea157f4..dc788d3ac 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ build-atenet: .PHONY: build-demos build-demos: - $(KO) build --ldflags="$(LDFLAGS)" ./demos/counter + $(KO) build --ldflags="$(LDFLAGS)" ./demos/counter ./demos/egress .PHONY: test test: diff --git a/cmd/atenet/internal/router/extproc.go b/cmd/atenet/internal/router/extproc.go index 1094a98c0..8e9944f5b 100644 --- a/cmd/atenet/internal/router/extproc.go +++ b/cmd/atenet/internal/router/extproc.go @@ -100,14 +100,6 @@ func (s *ExtProcServer) Process(stream extprocv3.ExternalProcessor_ProcessServer // the accepting listener, not by anything in the request itself // (see isEgressRequest). // - // SEE(lior): main grew a ResumeOutcome return on - // handleRequestHeaders while this branch was out. Rather than - // splitting the dispatch into two separately-typed call sites, the - // egress handler was widened to the same signature and returns - // ResumeOutcomeNone — egress requires an already-RUNNING actor and - // never resumes one, so "none" is accurate, and it keeps the - // route-duration metric's resume label a closed set (no new empty - // value) across both directions. handle := s.handleRequestHeaders if isEgressRequest(req) { handle = s.handleEgressRequestHeaders diff --git a/cmd/atenet/internal/router/extproc_egress.go b/cmd/atenet/internal/router/extproc_egress.go index 9649a6a2e..febafaae3 100644 --- a/cmd/atenet/internal/router/extproc_egress.go +++ b/cmd/atenet/internal/router/extproc_egress.go @@ -36,19 +36,29 @@ import ( ) const ( - // EgressListenerName is the Envoy listener that terminates actor egress - // CONNECTs. It must stay in sync with the listener name in - // manifests/ate-install/ateway-egress.yaml. - EgressListenerName = "egress" - // ListenerNameAttribute is the CEL attribute carrying the name of the - // listener that accepted the request. The egress Envoy asks for it via + // EgressFilterChainName is the Envoy filter chain that terminates actor + // egress CONNECTs. It must stay in sync with the filter chain name in + // manifests/ate-install/atenet-egress.yaml. + EgressFilterChainName = "egress" + // FilterChainNameAttribute is the CEL attribute carrying the name of the + // filter chain that accepted the request. The egress Envoy asks for it via // request_attributes on its ext_proc filter. - ListenerNameAttribute = "xds.listener_name" + // + // SEE(lior): this was xds.listener_name, which reads more naturally but + // which Envoy 1.34 cannot parse: it logs "error parsing cel expression + // xds.listener_name" at trace level, then sends the ProcessingRequest with + // an empty attributes map rather than failing config load. Because an + // absent attribute means "ingress" (the fail-safe direction), every egress + // CONNECT silently took the ingress path and 404'd on the actor DNS name + // parse. xds.filter_chain_name parses on the same Envoy build and is + // equally Envoy-asserted, so the trust model is unchanged. + FilterChainNameAttribute = "xds.filter_chain_name" // forwardedClientCertHeader is the header Envoy fills in with details of - // the mTLS peer, including the PEM chain it validated. The egress listener - // sets forward_client_cert_details: SANITIZE_SET, so whatever a client - // sends under this name is discarded and replaced by Envoy's own value. + // the mTLS peer, including the PEM chain it validated. The egress filter + // chain sets forward_client_cert_details: SANITIZE_SET, so whatever a + // client sends under this name is discarded and replaced by Envoy's own + // value. // // This is the only channel that can carry a whole certificate to ext_proc: // the CEL request attributes Envoy exposes (subject, SANs, SHA-256 digest) @@ -61,31 +71,31 @@ const ( ) // isEgressRequest reports whether an ext_proc RequestHeaders callback arrived on -// the egress gateway's listener rather than on an ingress listener. This lets -// one ext_proc server handle both directions off the same stream. +// the egress gateway's filter chain rather than on an ingress one. This lets one +// ext_proc server handle both directions off the same stream. // -// Dispatch is by listener, not by :method, because the two handlers apply +// Dispatch is by filter chain, not by :method, because the two handlers apply // opposite trust models: on egress the actor identity comes from a client // certificate Envoy validated, while on ingress every request header is // unauthenticated client input. Keying on :method would let any external client // sending CONNECT select the egress handler and use its denial messages as an -// actor-existence and status oracle. Envoy asserts the listener name; the +// actor-existence and status oracle. Envoy asserts the filter chain name; the // request cannot influence it. // // An unrecognized or absent attribute means ingress, the fail-safe direction: an // egress request misrouted to the ingress handler fails to parse as an actor DNS // name and 404s, whereas the reverse leaks control-plane state. func isEgressRequest(req *extprocv3.ProcessingRequest) bool { - return listenerName(req) == EgressListenerName + return filterChainName(req) == EgressFilterChainName } -// listenerName returns the xds.listener_name attribute Envoy attached to the -// request, or "" when the listener did not request the attribute. The +// filterChainName returns the xds.filter_chain_name attribute Envoy attached to +// the request, or "" when the listener did not request the attribute. The // attributes map is keyed by the ext_proc filter's name within the HCM chain, // which we do not want to hardcode here, so scan every entry. -func listenerName(req *extprocv3.ProcessingRequest) string { +func filterChainName(req *extprocv3.ProcessingRequest) string { for _, attrs := range req.GetAttributes() { - if v, ok := attrs.GetFields()[ListenerNameAttribute]; ok { + if v, ok := attrs.GetFields()[FilterChainNameAttribute]; ok { return v.GetStringValue() } } @@ -106,19 +116,19 @@ func listenerName(req *extprocv3.ProcessingRequest) string { // with a single branch. The (target, tmplNs, tmplName) results are unused for // egress and returned empty. // -// SEE(lior): the trailing ResumeOutcome exists only to match -// handleRequestHeaders after main added it. Egress never resumes an actor — it -// requires one already RUNNING — so every path returns ResumeOutcomeNone. +// The trailing ResumeOutcome exists only to match handleRequestHeaders. +// Egress never resumes an actor — it requires one already RUNNING - +// so every path returns ResumeOutcomeNone. func (s *ExtProcServer) handleEgressRequestHeaders( ctx context.Context, reqHeaders *extprocv3.HttpHeaders, ) (*extprocv3.HeadersResponse, *requestMetadata, string, string, string, ResumeOutcome, error) { metadata := newRequestMetadata(reqHeaders.Headers.GetHeaders()) - // Dispatch is by listener, so reaching here means the egress listener - // accepted the request. That listener only routes CONNECT (its sole route - // is a connect_matcher), so anything else is a config drift rather than a - // client the gateway should tunnel for. + // Dispatch is by filter chain, so reaching here means the egress chain + // accepted the request. That chain only routes CONNECT (its sole route is a + // connect_matcher), so anything else is config drift rather than a client + // the gateway should tunnel for. if !strings.EqualFold(metadata.method, "CONNECT") { return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_MethodNotAllowed, "egress denied: expected CONNECT, got %q", metadata.method) diff --git a/cmd/atenet/internal/router/extproc_egress_test.go b/cmd/atenet/internal/router/extproc_egress_test.go index 48ab0c0e9..41b193380 100644 --- a/cmd/atenet/internal/router/extproc_egress_test.go +++ b/cmd/atenet/internal/router/extproc_egress_test.go @@ -45,9 +45,9 @@ import ( ) // connectRequest builds a RequestHeaders ProcessingRequest for an egress -// CONNECT, optionally attributed to a listener. filterKey is the ext_proc +// CONNECT, optionally attributed to a filter chain. filterKey is the ext_proc // filter name Envoy keys the attributes map by. -func connectRequest(filterKey, listener string) *extprocv3.ProcessingRequest { +func connectRequest(filterKey, chain string) *extprocv3.ProcessingRequest { req := &extprocv3.ProcessingRequest{ Request: &extprocv3.ProcessingRequest_RequestHeaders{ RequestHeaders: &extprocv3.HttpHeaders{ @@ -60,11 +60,11 @@ func connectRequest(filterKey, listener string) *extprocv3.ProcessingRequest { }, }, } - if listener != "" { + if chain != "" { req.Attributes = map[string]*structpb.Struct{ filterKey: { Fields: map[string]*structpb.Value{ - ListenerNameAttribute: structpb.NewStringValue(listener), + FilterChainNameAttribute: structpb.NewStringValue(chain), }, }, } @@ -76,19 +76,19 @@ func TestIsEgressRequest(t *testing.T) { tests := []struct { name string filterKey string - listener string + chain string want bool }{ { - name: "egress listener", + name: "egress filter chain", filterKey: "envoy.filters.http.ext_proc", - listener: EgressListenerName, + chain: EgressFilterChainName, want: true, }, { - name: "egress listener under a renamed filter", + name: "egress filter chain under a renamed filter", filterKey: "some.custom.ext_proc.name", - listener: EgressListenerName, + chain: EgressFilterChainName, want: true, }, { @@ -98,33 +98,33 @@ func TestIsEgressRequest(t *testing.T) { // exists and is running. name: "CONNECT on the ingress HTTP listener", filterKey: "envoy.filters.http.ext_proc", - listener: IngressHTTPListener, + chain: IngressHTTPListener, want: false, }, { name: "CONNECT on the ingress HTTPS listener", filterKey: "envoy.filters.http.ext_proc", - listener: IngressHTTPSListener, + chain: IngressHTTPSListener, want: false, }, { // A listener that never requested the attribute falls back to // ingress, the fail-safe direction. - name: "no attributes at all", - listener: "", - want: false, + name: "no attributes at all", + chain: "", + want: false, }, { - name: "unrecognised listener", + name: "unrecognised filter chain", filterKey: "envoy.filters.http.ext_proc", - listener: "some-other-listener", + chain: "some-other-chain", want: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - if got := isEgressRequest(connectRequest(tc.filterKey, tc.listener)); got != tc.want { + if got := isEgressRequest(connectRequest(tc.filterKey, tc.chain)); got != tc.want { t.Errorf("isEgressRequest() = %v, want %v", got, tc.want) } }) @@ -132,17 +132,17 @@ func TestIsEgressRequest(t *testing.T) { } // A request the client dresses up to look like egress must not be enough: only -// the Envoy-asserted listener name selects the egress handler. +// the Envoy-asserted filter chain name selects the egress handler. func TestIsEgressRequestIgnoresClientSuppliedAttributeHeader(t *testing.T) { req := connectRequest("envoy.filters.http.ext_proc", IngressHTTPListener) rh := req.GetRequestHeaders().GetHeaders() rh.Headers = append(rh.Headers, - &corev3.HeaderValue{Key: ListenerNameAttribute, RawValue: []byte(EgressListenerName)}, - &corev3.HeaderValue{Key: "x-envoy-listener-name", RawValue: []byte(EgressListenerName)}, + &corev3.HeaderValue{Key: FilterChainNameAttribute, RawValue: []byte(EgressFilterChainName)}, + &corev3.HeaderValue{Key: "x-envoy-filter-chain-name", RawValue: []byte(EgressFilterChainName)}, ) if isEgressRequest(req) { - t.Error("isEgressRequest() = true for a client-forged listener header, want false") + t.Error("isEgressRequest() = true for a client-forged filter chain header, want false") } } @@ -290,7 +290,7 @@ func xfccHeaderDER(chain ...[]byte) string { for _, der := range chain { _ = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}) } - return fmt.Sprintf(`By=spiffe://cluster.local/ns/ate-system/sa/ateway-egress;Hash=abc123;Chain="%s"`, + return fmt.Sprintf(`By=spiffe://cluster.local/ns/ate-system/sa/atenet-egress;Hash=abc123;Chain="%s"`, url.PathEscape(buf.String())) } @@ -539,7 +539,7 @@ func TestHandleEgressRequestHeadersRejectsBadCertificates(t *testing.T) { { name: "XFCC without a Chain value", xfcc: func(*testing.T) string { - return `By=spiffe://cluster.local/ns/ate-system/sa/ateway-egress;Hash=abc123` + return `By=spiffe://cluster.local/ns/ate-system/sa/atenet-egress;Hash=abc123` }, want: envoy_type.StatusCode_Forbidden, }, diff --git a/demos/egress/README.md b/demos/egress/README.md new file mode 100644 index 000000000..9360b40c1 --- /dev/null +++ b/demos/egress/README.md @@ -0,0 +1,148 @@ +# Egress Demo — Pluggable Egress Networking + +This demo shows an Actor's outbound traffic being **transparently tunneled through an +egress gateway** and **authenticated by actor identity**, end to end. + +The Actor is a tiny service that accepts `{"url":"..."}`, performs an HTTP `GET`, and returns +the upstream response. The Actor believes it is dialing plain HTTP directly — but its egress is +intercepted and carried over mTLS to a gateway that verifies who is making the request. + +## What it demonstrates + +``` + ┌──────────────── ateom worker pod ─────────────────┐ + │ Actor (gVisor) │ + │ GET http://:80/ (plain HTTP) │ + │ │ │ + │ ▼ nftables REDIRECT │ + │ atunnel egress ──(mTLS with the actor's own │ + │ │ certificate + bare CONNECT) │ + └────────┼────────────────────────────────────────────┘ + ▼ + ┌──────────── atenet-egress pod ───────────────────┐ + │ Envoy egress gateway │ + │ • downstream mTLS, trusted_ca = actor-id CA │ + │ • terminates HTTP CONNECT │ + │ • ext_proc ──(localhost)──► atenet router (ext_proc sidecar) + │ (forwards the peer chain │ verify chain + ActorIdentity extension + │ as x-forwarded-client-cert)│ GetActor → UID must match, must be RUNNING + │ • dynamic_forward_proxy │ allow / deny 403 + │ │ + └───────────┼───────────────────────────────────────┘ + ▼ + real destination (the CONNECT authority, an IP:port) +``` + +1. **Guide 1 — gateway accepts CONNECT + mTLS.** `atenet-egress` is an Envoy dynamic-forward-proxy + that terminates the actor's mTLS `CONNECT` and tunnels to the requested destination. +2. **Guide 2 — transparent interception.** `nftables` REDIRECTs actor TCP egress into `atunnel`, + which wraps it in mTLS + `CONNECT`. +3. **Guide 3 — HTTP-only actors, identity carried by the certificate.** The Actor only dials plain + HTTP. atunnel presents the actor's own certificate — minted per actor by ateapi off the + actor-identity CA, carrying an `ActorIdentity` X.509 extension — and sends a bare `CONNECT` + with no identity headers at all. +4. **Identity authentication.** Envoy requires a client certificate signed by the actor-identity + CA, so a non-actor client is refused at the handshake. It then forwards the verified chain to + `ext_proc` as `x-forwarded-client-cert`, and the **atenet router** (co-located in the gateway + pod as an ext_proc sidecar, the same ext_proc code used for ingress) re-verifies the chain, + requires exactly one `ActorIdentity` extension with `purpose: atunnel`, and calls the ate API + (`GetActor`). It returns **403** unless the certified **UID** matches a real, `RUNNING` actor. + This mirrors the ingress gateway's Envoy + ext_proc co-location; a standalone/shared ext_proc + is a future step. + +## Components + +- **Egress app (`main.go`)** — the Actor: `POST /` with `{"url":"..."}` → fetches it → returns + status + body. +- **Egress gateway** — `manifests/ate-install/atenet-egress.yaml`. One pod, two containers: + an Envoy (`envoy`) and the atenet router ext_proc (`ext-proc`), called over localhost. +- **Egress opt-in** — `ate-api-server --egress-gateway-address=atenet-egress.ate-system.svc:443` + (set in `manifests/ate-install/ate-api-server.yaml`). ateapi stamps the address onto every + atelet `Run`/`Restore`, which turns on tunneled egress cluster-wide. +- **Actor-identity trust** — the gateway mounts the `actor-id-ca-certs` Secret, a cert-only copy of + the actor-identity CA root that `hack/install-ate.sh` derives from `actor-id-ca-pool` (which also + holds the CA signing key and is deliberately *not* mounted here). + +## Prerequisites + +- A kind cluster with Agent Substrate installed (`hack/create-kind-cluster.sh` then + `hack/install-ate-kind.sh --deploy-ate-system`). Egress is enabled by the ateapi flag above. +- `ko`, `kubectl`, and `kubectl-ate` (`go install ./cmd/kubectl-ate`). + +## Deploy the demo fixture + +```bash +./hack/install-ate.sh --deploy-demo-egress +kubectl wait --for=condition=Ready actortemplate/egress -n ate-demo-egress --timeout=5m +``` + +## Run the automated test (easiest) + +```bash +./demos/egress/test-egress.sh +``` + +It deploys an in-cluster HTTP target, creates & resumes an Actor, then asserts: + +- **positive** — a real Actor's egress reaches the target (`HTTP 200`) *through the gateway* + (the target sees the gateway's IP as its client), and the gateway logs the CONNECT against the + actor's certificate SAN; +- **negative** — a pod holding a valid *pod* identity but no actor certificate cannot open a + tunnel at all: the gateway's `trusted_ca` is the actor-identity CA, so the mTLS handshake is + refused before any CONNECT is answered. + +Add `--cleanup` to remove everything the script created. + +## Manual walkthrough + +```bash +# 1. An in-cluster target the Actor will fetch (any HTTP server works). +kubectl create namespace egress-target +kubectl -n egress-target create deployment whoami --image=traefik/whoami +kubectl -n egress-target expose deployment whoami --port=80 +TARGET_IP=$(kubectl -n egress-target get svc whoami -o jsonpath='{.spec.clusterIP}') + +# 2. Create and resume an Actor. +kubectl ate create atespace demo +kubectl ate create actor egress-demo -a demo --template ate-demo-egress/egress +kubectl ate resume actor egress-demo -a demo # wait for STATUS_RUNNING + +# 3. Drive the Actor's egress through the ingress gateway. +kubectl -n ate-system port-forward service/atenet-router 8000:80 & +curl -s -X POST http://localhost:8000/ \ + -H 'Host: egress-demo.demo.actors.resources.substrate.ate.dev' \ + -H 'Content-Type: application/json' \ + -d "{\"url\":\"http://${TARGET_IP}:80/\"}" +``` + +### What to observe + +```bash +# The egress gateway logs each tunneled CONNECT against the verified peer certificate: +kubectl -n ate-system logs deploy/atenet-egress | grep '\[egress\]' +# [egress] authority=:80 peer_san=spiffe://substrate-actor.local/atespace/demo/actor/egress-demo … code=200 … + +# The co-located ext_proc sidecar logs the identity decision, including the UID it authorized on: +kubectl -n ate-system logs deploy/atenet-egress -c ext-proc | grep -i 'egress identity\|egress denied' +# egress identity authenticated atespace=demo actor=egress-demo actorUid=… status=STATUS_RUNNING +``` + +The `whoami` body shows `RemoteAddr: ` — proof the request egressed +*through* the gateway rather than directly. + +## Notes / limitations + +- This milestone **authenticates** identity (is this a real, running actor?). **Authorizing** + egress by destination and injecting upstream credentials/tokens is a follow-up, implemented in + the same `ext_proc` (policy API TBD). +- Identity comes entirely from the actor certificate: the atespace, actor name, and UID are read + out of the `ActorIdentity` extension and the UID is matched against the live actor, so a + certificate cannot survive its actor being deleted and recreated under the same name. Nothing + the actor can write into the CONNECT contributes to the decision. + +## Cleanup + +```bash +./demos/egress/test-egress.sh --cleanup +./hack/install-ate.sh --delete-demo-egress +``` diff --git a/demos/egress/egress.yaml.tmpl b/demos/egress/egress.yaml.tmpl new file mode 100644 index 000000000..e6f9b080f --- /dev/null +++ b/demos/egress/egress.yaml.tmpl @@ -0,0 +1,56 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-demo-egress + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: egress + namespace: ate-demo-egress + labels: + workload: egress +spec: + replicas: 2 + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: egress + namespace: ate-demo-egress +spec: + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + containers: + - name: egress + image: ko://github.com/agent-substrate/substrate/demos/egress + command: ["/ko-app/egress"] + readyz: + httpGet: + path: /readyz + port: 80 + workerSelector: + matchLabels: + workload: egress + snapshotsConfig: + onPause: Full + onCommit: Full + location: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/egress/main.go b/demos/egress/main.go new file mode 100644 index 000000000..4c0288e12 --- /dev/null +++ b/demos/egress/main.go @@ -0,0 +1,125 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command egress is a small HTTP service for demonstrating per-Actor egress +// policy. It accepts a URL, fetches it, and returns the upstream response. +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "time" +) + +const ( + listenAddress = ":80" + maxRequestBody = 64 << 10 + maxResponseBody = 1 << 20 + requestTimeout = 15 * time.Second +) + +type fetchRequest struct { + URL string `json:"url"` +} + +type fetchResponse struct { + StatusCode int `json:"statusCode,omitempty"` + Body string `json:"body,omitempty"` + Error string `json:"error,omitempty"` +} + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + + client := &http.Client{Timeout: requestTimeout} + slog.Info("starting egress demo", "address", listenAddress) + if err := http.ListenAndServe(listenAddress, newHandler(client)); err != nil { + slog.Error("egress demo stopped", "error", err) + os.Exit(1) + } +} + +func newHandler(client *http.Client) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "ok\n") + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + writeJSON(w, http.StatusMethodNotAllowed, fetchResponse{Error: "method must be POST"}) + return + } + + var input fetchRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxRequestBody)) + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("invalid JSON payload: %v", err)}) + return + } + if err := validateURL(input.URL); err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: err.Error()}) + return + } + + outbound, err := http.NewRequestWithContext(r.Context(), http.MethodGet, input.URL, nil) + if err != nil { + writeJSON(w, http.StatusBadRequest, fetchResponse{Error: fmt.Sprintf("invalid URL: %v", err)}) + return + } + if traceparent := r.Header.Get("traceparent"); traceparent != "" { + outbound.Header.Set("traceparent", traceparent) + } + response, err := client.Do(outbound) + if err != nil { + writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("request failed: %v", err)}) + return + } + defer response.Body.Close() + + body, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBody)) + if err != nil { + writeJSON(w, http.StatusBadGateway, fetchResponse{Error: fmt.Sprintf("reading response: %v", err)}) + return + } + writeJSON(w, response.StatusCode, fetchResponse{StatusCode: response.StatusCode, Body: string(body)}) + }) + return mux +} + +func validateURL(raw string) error { + parsed, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return fmt.Errorf("URL scheme must be http or https") + } + if parsed.Hostname() == "" { + return fmt.Errorf("URL must include a hostname") + } + return nil +} + +func writeJSON(w http.ResponseWriter, status int, response fetchResponse) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(response) +} diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go new file mode 100644 index 000000000..6cd203732 --- /dev/null +++ b/demos/egress/main_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestFetch(t *testing.T) { + const traceparent = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodGet { + t.Errorf("upstream method = %s, want GET", r.Method) + } + if got := r.Header.Get("traceparent"); got != traceparent { + t.Errorf("upstream traceparent = %q, want %q", got, traceparent) + } + return &http.Response{ + StatusCode: http.StatusTeapot, + Body: io.NopCloser(strings.NewReader("hello from upstream")), + Header: make(http.Header), + }, nil + })} + + payload, err := json.Marshal(fetchRequest{URL: "https://allowed.example/"}) + if err != nil { + t.Fatal(err) + } + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(payload))) + request.Header.Set("traceparent", traceparent) + newHandler(client).ServeHTTP(recorder, request) + + if recorder.Code != http.StatusTeapot { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusTeapot) + } + var got fetchResponse + if err := json.NewDecoder(recorder.Body).Decode(&got); err != nil { + t.Fatalf("decoding response: %v", err) + } + if got.StatusCode != http.StatusTeapot || got.Body != "hello from upstream" { + t.Errorf("response = %+v", got) + } +} + +func TestInvalidRequests(t *testing.T) { + tests := []struct { + name string + method string + body string + status int + }{ + {name: "method", method: http.MethodGet, body: `{}`, status: http.StatusMethodNotAllowed}, + {name: "malformed JSON", method: http.MethodPost, body: `{`, status: http.StatusBadRequest}, + {name: "missing hostname", method: http.MethodPost, body: `{"url":"https:///path"}`, status: http.StatusBadRequest}, + {name: "unsupported scheme", method: http.MethodPost, body: `{"url":"file:///etc/passwd"}`, status: http.StatusBadRequest}, + } + + handler := newHandler(http.DefaultClient) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(test.method, "/", strings.NewReader(test.body)) + handler.ServeHTTP(recorder, request) + if recorder.Code != test.status { + t.Errorf("status = %d, want %d; body = %s", recorder.Code, test.status, recorder.Body.String()) + } + }) + } +} + +func TestOutboundFailure(t *testing.T) { + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("blocked") + })} + handler := newHandler(client) + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"url":"https://example.com/"}`)) + + handler.ServeHTTP(recorder, request) + + if recorder.Code != http.StatusBadGateway { + t.Errorf("status = %d, want %d; body = %s", recorder.Code, http.StatusBadGateway, recorder.Body.String()) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh new file mode 100755 index 000000000..7566146a1 --- /dev/null +++ b/demos/egress/test-egress.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# End-to-end test for pluggable actor egress. Reproduces: +# * POSITIVE — a real, running Actor's plain-HTTP egress is transparently +# tunneled (nftables -> atunnel -> mTLS + CONNECT) through the Envoy egress +# gateway to an in-cluster target, and the gateway's ext_proc authenticates +# the actor certificate against the ate API (allowed, HTTP 200). +# * NEGATIVE — a pod holding a perfectly valid *pod* identity, but no actor +# certificate, cannot open a tunnel: the gateway's trusted_ca is the +# actor-identity CA, so the mTLS handshake itself is refused. +# +# Prerequisites: a substrate cluster with `--deploy-demo-egress` applied, plus +# kubectl and kubectl-ate on PATH. See demos/egress/README.md. +# +# Usage: +# demos/egress/test-egress.sh # run the tests +# demos/egress/test-egress.sh --cleanup # remove everything this script created + +set -o errexit -o nounset -o pipefail + +CTX="${KUBECTL_CONTEXT:-kind-kind}" +ATESPACE="${ATESPACE:-demo}" +ACTOR="${ACTOR:-egress-demo}" +TEMPLATE="${TEMPLATE:-ate-demo-egress/egress}" +TARGET_NS="${TARGET_NS:-egress-target}" +PROBE_POD="egress-identity-probe" + +K="kubectl --context ${CTX}" +KATE="kubectl-ate --context ${CTX}" + +log() { printf '\n\033[1;36m== %s\033[0m\n' "$*"; } +info() { printf ' %s\n' "$*"; } +pass() { printf '\033[1;32mPASS\033[0m %s\n' "$*"; } +fail() { printf '\033[1;31mFAIL\033[0m %s\n' "$*"; FAILED=1; } +FAILED=0 + +require() { command -v "$1" >/dev/null 2>&1 || { echo "missing required tool: $1"; exit 1; }; } + +cleanup() { + log "cleanup" + ${K} -n ate-system delete pod "${PROBE_POD}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + ${KATE} suspend actor "${ACTOR}" -a "${ATESPACE}" >/dev/null 2>&1 || true + ${KATE} delete actor "${ACTOR}" -a "${ATESPACE}" >/dev/null 2>&1 || true + ${K} delete namespace "${TARGET_NS}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + info "done" +} + +if [[ "${1:-}" == "--cleanup" ]]; then require kubectl; require kubectl-ate; cleanup; exit 0; fi + +require kubectl +require kubectl-ate +trap '[[ "${KEEP:-}" == "1" ]] || cleanup' EXIT + +log "preflight: egress gateway (Envoy + co-located ext_proc) is running" +${K} -n ate-system rollout status deployment/atenet-egress --timeout=120s + +log "deploy an in-cluster HTTP target (whoami)" +${K} create namespace "${TARGET_NS}" >/dev/null 2>&1 || true +${K} -n "${TARGET_NS}" create deployment whoami --image=traefik/whoami >/dev/null 2>&1 || true +${K} -n "${TARGET_NS}" expose deployment whoami --port=80 >/dev/null 2>&1 || true +${K} -n "${TARGET_NS}" rollout status deployment/whoami --timeout=120s +TARGET_IP=$(${K} -n "${TARGET_NS}" get svc whoami -o jsonpath='{.spec.clusterIP}') +info "target ClusterIP = ${TARGET_IP}" + +log "create + resume Actor ${ATESPACE}/${ACTOR}" +${KATE} create atespace "${ATESPACE}" >/dev/null 2>&1 || true +${KATE} create actor "${ACTOR}" -a "${ATESPACE}" --template "${TEMPLATE}" >/dev/null 2>&1 || true +${KATE} resume actor "${ACTOR}" -a "${ATESPACE}" >/dev/null 2>&1 || true +for _ in $(seq 1 30); do + ${KATE} get actors -a "${ATESPACE}" 2>/dev/null | grep -q "STATUS_RUNNING" && break + sleep 3 +done +${KATE} get actors -a "${ATESPACE}" 2>/dev/null | grep "${ACTOR}" || true +${KATE} get actors -a "${ATESPACE}" 2>/dev/null | grep -q "STATUS_RUNNING" || { echo "actor did not reach RUNNING"; exit 1; } + +egress_log_since() { ${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null | grep '\[egress\]' | tail -n +"$(( $1 + 1 ))"; } +egress_log_count() { ${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null | grep -c '\[egress\]' || true; } +# wait_egress_log : retry for log-shipping lag. +wait_egress_log() { for _ in $(seq 1 10); do if egress_log_since "$1" | grep -qE "$2"; then egress_log_since "$1" | grep -E "$2" | tail -1; return 0; fi; sleep 1; done; return 1; } + +############################################################################## +log "POSITIVE — real Actor egress is tunneled through the gateway (expect 200)" +############################################################################## +BEFORE=$(egress_log_count) +${K} -n ate-system port-forward service/atenet-router 18099:80 >/tmp/egress-pf.log 2>&1 & +PF=$!; sleep 4 +CODE=$(curl -s -o /tmp/egress-body.txt -w '%{http_code}' -X POST http://localhost:18099/ \ + -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H 'Content-Type: application/json' \ + -d "{\"url\":\"http://${TARGET_IP}:80/\"}" || true) +kill "${PF}" >/dev/null 2>&1 || true +sleep 1 +info "actor round-trip HTTP ${CODE}" +GW_IP=$(${K} -n ate-system get pod -l app=atenet-egress -o jsonpath='{.items[0].status.podIP}') +if [[ "${CODE}" == "200" ]]; then pass "actor fetched the target (HTTP 200)"; else fail "expected HTTP 200, got ${CODE}"; fi +if grep -q "RemoteAddr: ${GW_IP}" /tmp/egress-body.txt 2>/dev/null; then + pass "target saw the egress gateway (${GW_IP}) as its client — traffic went through the gateway" +else + info "target body RemoteAddr: $(grep -o 'RemoteAddr: [0-9.]*' /tmp/egress-body.txt 2>/dev/null || echo '?') (gateway IP ${GW_IP})" +fi +# The access log identifies the peer by its certificate SAN +# (spiffe://substrate-actor.local/atespace//actor/), not by any +# header the actor could have written. +if LINE=$(wait_egress_log "${BEFORE}" "actor/${ACTOR}.*code=200"); then + pass "gateway logged the CONNECT: ${LINE}" +else + fail "gateway did not log an allowed CONNECT for ${ACTOR}" +fi + +############################################################################## +log "NEGATIVE — a pod identity is not an actor identity (expect a refused handshake)" +############################################################################## +# SEE(lior): this used to be a 403 assertion. On the header-based design the +# gateway trusted any podidentity holder's mTLS and let ext_proc adjudicate the +# X-Ate-* headers it asserted, so a probe pod could reach the CONNECT and be +# denied there. The gateway now trusts only the actor-identity CA, so the same +# probe never gets past the TLS handshake — the denial moved a layer down and +# there is no CONNECT response to read a status code out of. Assert the +# handshake failure instead; it is the stronger property. +${K} apply -f - >/dev/null <<'YAML' +apiVersion: v1 +kind: Pod +metadata: + name: egress-identity-probe + namespace: ate-system +spec: + containers: + - name: curl + image: curlimages/curl:latest + command: ["sleep", "600"] + volumeMounts: + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev, readOnly: true } + volumes: + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - name: servicedns + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: { matchLabels: { podcert.ate.dev/canarying: live } } + path: trust-bundle.pem +YAML +${K} -n ate-system wait --for=condition=Ready pod/${PROBE_POD} --timeout=60s >/dev/null + +BEFORE=$(egress_log_count) +# %{http_connect} carries the proxy's CONNECT response code, and stays 000 when +# the tunnel never opens. curl exits non-zero on a failed handshake, so capture +# both and require that no CONNECT was ever answered. +PROBE=$(${K} -n ate-system exec ${PROBE_POD} -- sh -c "curl -s -o /dev/null -w '%{http_connect}' \ + --proxy-cacert /run/servicedns.podcert.ate.dev/trust-bundle.pem \ + --proxy-cert /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --proxy-key /run/podidentity.podcert.ate.dev/credential-bundle.pem \ + --proxytunnel -x https://atenet-egress.ate-system.svc:443 http://${TARGET_IP}:80/; echo \" exit=\$?\"" || true) +CODE=${PROBE%% *} +info "pod-identity CONNECT attempt: http_connect=${CODE:-000}${PROBE#"${CODE}"}" +if [[ "${CODE}" != "200" ]]; then + pass "a pod identity cannot open an egress tunnel (no CONNECT succeeded)" +else + fail "expected the gateway to refuse a non-actor client certificate, but CONNECT returned 200" +fi +# A rejected handshake never becomes an HTTP request, so it produces no [egress] +# access-log line — only a connection-level TLS error. Report whatever the +# gateway logged so a real failure is diagnosable, but do not assert on it. +if LINE=$(egress_log_since "${BEFORE}" | tail -1); [[ -n "${LINE:-}" ]]; then + info "gateway egress log since the probe: ${LINE}" +else + info "no new [egress] access-log lines — the handshake was refused before HTTP, as expected" +fi + +echo +if [[ "${FAILED}" == "0" ]]; then + printf '\033[1;32mALL CHECKS PASSED\033[0m — pluggable egress + identity authentication working.\n' +else + printf '\033[1;31mSOME CHECKS FAILED\033[0m\n'; exit 1 +fi diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 7a6c1471c..5154b1032 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -40,6 +40,7 @@ ATE_DEMOS=() # Include demos. source "${ROOT}"/hack/install-demo-counter.sh +source "${ROOT}"/hack/install-demo-egress.sh source "${ROOT}"/hack/install-demo-sandbox.sh source "${ROOT}"/hack/install-demo-claude-code-multiplex.sh source "${ROOT}"/hack/install-demo-multi-template.sh @@ -446,7 +447,7 @@ deploy_ate_system() { run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout=120s run_kubectl rollout status deployment/ate-controller -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s - run_kubectl rollout status deployment/ateway-egress -n ate-system --timeout=120s + run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s run_kubectl rollout status statefulset/valkey-cluster -n ate-system --timeout=120s run_kubectl rollout status daemonset/atelet -n ate-system --timeout=120s } @@ -521,15 +522,10 @@ deploy_atenet() { router_manifest="$(render_atenet_router_manifest)" echo "${router_manifest}" | run_kubectl apply -f - - run_ko apply -f manifests/ate-install/ateway-egress.yaml + run_ko apply -f manifests/ate-install/atenet-egress.yaml run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s - run_kubectl rollout status deployment/ateway-egress -n ate-system --timeout=120s - # SEE(lior): this branch also added a `deployment/atenet-dns` wait here, which - # main has since fixed to `deployment/dns` (the Deployment in atenet-dns.yaml - # is named "dns"; every other resource in that file is "atenet-dns", so the - # old name was NotFound on every successful deploy). Dropped this branch's - # duplicate in favour of main's corrected line. + run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s run_kubectl rollout status deployment/dns -n ate-system --timeout=120s } @@ -648,7 +644,7 @@ delete_atenet() { run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-router.yaml run_kubectl delete --ignore-not-found \ -f manifests/ate-install/components/agentgateway/configmap.yaml - run_kubectl delete --ignore-not-found -f manifests/ate-install/ateway-egress.yaml + run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-egress.yaml run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-dns.yaml } diff --git a/hack/install-demo-egress.sh b/hack/install-demo-egress.sh new file mode 100644 index 000000000..e4e0074a9 --- /dev/null +++ b/hack/install-demo-egress.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is sourced as part of install-ate.sh. Do not run directly. + +ATE_DEMOS+=(demo-egress) # register demo-egress + +demo-egress_cmdline() { + case "${1}" in + --deploy-demo-egress) demo-egress_deploy ;; + --delete-demo-egress) demo-egress_delete ;; + *) + return 1 + ;; + esac + return 0 +} + +demo-egress_deploy() { + log_step "demo-egress_deploy" + ensure_crds + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" demos/egress/egress.yaml.tmpl \ + | run_ko apply -f - + + log_step "Waiting for egress demo to be ready..." + # The WorkerPool controller names the Deployment after the WorkerPool + # ("egress"), the same way demo-counter gets "deployment/counter". The old + # "egress-deployment" name was NotFound on every successful deploy. + run_kubectl rollout status deployment/egress -n ate-demo-egress --timeout=300s + run_kubectl wait --for=condition=Ready actortemplate/egress -n ate-demo-egress --timeout=300s +} + +demo-egress_delete() { + log_step "demo-egress_delete" + delete_demo_actors ate-demo-egress egress + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" demos/egress/egress.yaml.tmpl \ + | run_kubectl delete --ignore-not-found -f - +} diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh new file mode 100644 index 000000000..e1fadf543 --- /dev/null +++ b/hack/verify-egress-demo.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Preconditions: +# hack/create-kind-cluster.sh +# hack/install-ate-kind.sh --deploy-demo-egress +# +# The egress demo Actor accepts {"url":"..."} and performs an HTTP GET. With +# egress turned on (ate-api-server --egress-gateway-address, which ateapi stamps +# onto every atelet Run/Restore), the Actor's outbound TCP is nftables-REDIRECTed +# into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the Envoy egress +# gateway, which terminates CONNECT and tunnels to the real destination. This +# script drives that path and shows the gateway's access log proving the actor's +# client certificate + CONNECT authority were seen. +set -o errexit -o nounset -o pipefail +ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}" + +CTX="${KUBECTL_CONTEXT:-kind-kind}" +K="kubectl --context ${CTX}" +ATESPACE="${ATESPACE:-demo}" +ACTOR="${ACTOR:-egress-demo}" +TARGET_URL="${TARGET_URL:-http://example.com/}" + +echo "== gateway should be running ==" +${K} -n ate-system rollout status deployment/atenet-egress --timeout=120s + +echo "== create atespace + actor ==" +kubectl-ate --context "${CTX}" create atespace "${ATESPACE}" 2>/dev/null || true +kubectl-ate --context "${CTX}" create actor "${ACTOR}" \ + --atespace "${ATESPACE}" --template ate-demo-egress/egress 2>/dev/null || true +${K} -n ate-system wait --for=condition=Ready "actor/${ACTOR}" 2>/dev/null || sleep 10 + +echo "== snapshot gateway log offset ==" +BEFORE=$(${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null | wc -l | tr -d ' ') + +echo "== drive actor egress: GET ${TARGET_URL} via the actor ==" +${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 & +PF=$!; trap 'kill ${PF} 2>/dev/null || true' EXIT +sleep 3 +RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ + -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H 'Content-Type: application/json' \ + -d "{\"url\":\"${TARGET_URL}\"}") || true +echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" + +echo "== NEW egress gateway access log lines (proof of CONNECT+mTLS+identity) ==" +${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null \ + | tail -n +"$((BEFORE + 1))" | grep '\[egress\]' || { + echo "!! no [egress] lines — dumping recent gateway logs:" + ${K} -n ate-system logs deployment/atenet-egress --tail=20 + exit 1 + } +echo "== PASS: actor egress traversed the Envoy egress gateway ==" diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 058e58852..f168e397e 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -15,8 +15,10 @@ package e2e import ( + "bytes" "context" "fmt" + "io" "net/http" "time" @@ -31,7 +33,7 @@ const ( routerService = "atenet-router" ) -// RouterClient sends HTTP requests to actors through the atenet router, the +// RouterClient sends HTTP requests to actors through the ingress atenet-router, the // same way real traffic arrives (so the request is routed and, if needed, the // actor is resumed). It port-forwards the router Service, mirroring the // approach in internal/ateclient. @@ -41,7 +43,7 @@ type RouterClient struct { stop func() } -// NewRouterClient establishes a port-forward to the atenet router. Call Close +// NewRouterClient establishes a port-forward to the ingress atenet-router. Call Close // to tear it down. func NewRouterClient(ctx context.Context) (*RouterClient, error) { config, err := ateclient.LoadConfig(KubeConfig, KubeContext) @@ -73,10 +75,23 @@ func (c *RouterClient) Close() { // Get issues GET path to actor through the router, setting the actor's mesh Host // so the router routes (and resumes) it. The caller must close the body. func (c *RouterClient) Get(ctx context.Context, actorRef resources.ActorRef, path string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil) + return c.request(ctx, http.MethodGet, actorRef, path, nil) +} + +// PostJSON issues a POST with a JSON body to an Actor through the router. The +// caller must close the response body. +func (c *RouterClient) PostJSON(ctx context.Context, actorRef resources.ActorRef, path string, body []byte) (*http.Response, error) { + return c.request(ctx, http.MethodPost, actorRef, path, bytes.NewReader(body)) +} + +func (c *RouterClient) request(ctx context.Context, method string, actorRef resources.ActorRef, path string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) if err != nil { return nil, err } + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/json") + } // The router routes on the Host/:authority, not a header. req.Host = actorRef.DNSName() return c.http.Do(req) diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go new file mode 100644 index 000000000..d9be88241 --- /dev/null +++ b/internal/e2e/router_client_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/agent-substrate/substrate/internal/resources" +) + +// SEE(lior): this file used to also hold TestResolveHTTPTargetPort and +// TestIsPodReady. Main moved both into internal/portforward/portforward_test.go +// (upstream 4453b5e7, "Prefactoring: Consolidate logic to port-forward to a +// service Pod") and deleted this file, so git rename-matched this branch's +// version of it against the new portforward test and reported the whole thing as +// a conflict. Took main's move as-is and re-added only the net-new PostJSON test +// here, rather than resurrecting the port-forward tests in a package that no +// longer owns that code. +func TestRouterClientPostJSON(t *testing.T) { + client := &RouterClient{ + baseURL: "http://router.test", + http: &http.Client{Transport: testRoundTripper(func(request *http.Request) (*http.Response, error) { + if request.Method != http.MethodPost { + t.Errorf("method = %q, want POST", request.Method) + } + if request.Host != "fetcher.demo.actors.resources.substrate.ate.dev" { + t.Errorf("host = %q", request.Host) + } + if request.URL.Path != "/fetch" { + t.Errorf("path = %q, want /fetch", request.URL.Path) + } + if request.Header.Get("Content-Type") != "application/json" { + t.Errorf("content type = %q, want application/json", request.Header.Get("Content-Type")) + } + body, err := io.ReadAll(request.Body) + if err != nil { + t.Fatalf("reading body: %v", err) + } + if string(body) != `{"url":"https://example.com/"}` { + t.Errorf("body = %q", body) + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("ok")), + Header: make(http.Header), + }, nil + })}, + } + + actorRef := resources.ActorRef{Atespace: "demo", Name: "fetcher"} + response, err := client.PostJSON(context.Background(), actorRef, "/fetch", []byte(`{"url":"https://example.com/"}`)) + if err != nil { + t.Fatalf("PostJSON: %v", err) + } + response.Body.Close() +} + +type testRoundTripper func(*http.Request) (*http.Response, error) + +func (f testRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/internal/e2e/suites/networking/networking_test.go b/internal/e2e/suites/networking/networking_test.go index 1cb32d54b..b4a919d9e 100644 --- a/internal/e2e/suites/networking/networking_test.go +++ b/internal/e2e/suites/networking/networking_test.go @@ -29,9 +29,24 @@ import ( const networkingAtespace = "networking-e2e" +// actorTemplate identifies a demo ActorTemplate to build test Actors from, +// along with the hack/install-ate.sh flag that deploys it. +type actorTemplate struct { + namespace string + name string + // deployFlag names the install flag that creates the template, so a + // missing fixture reports how to fix it rather than just failing. + deployFlag string +} + +var ( + counterTemplate = actorTemplate{namespace: "ate-demo-counter", name: "counter", deployFlag: "--deploy-demo-counter"} + egressTemplate = actorTemplate{namespace: "ate-demo-egress", name: "egress", deployFlag: "--deploy-demo-egress"} +) + func TestActorDirectAccess(t *testing.T) { ctx := context.Background() - actorName, actor := createAndResumeActor(t, ctx, "direct") + actorName, actor := createAndResumeActor(t, ctx, "direct", counterTemplate) router := mustRouterClient(t, ctx) defer router.Close() @@ -68,7 +83,38 @@ func TestActorDirectAccess(t *testing.T) { }) } -func createAndResumeActor(t *testing.T, ctx context.Context, prefix string) (string, *ateapipb.Actor) { +// TestActorEgress exercises the full egress path. The Actor's outbound TCP +// connection is transparently redirected by nftables into atunnel, wrapped in +// mTLS with the Actor's own actor-identity certificate plus an HTTP CONNECT to +// atenet-egress, authorized there against that certificate, and only then +// dialed out. A masqueraded (pre-gateway) egress would also return 200, so this +// asserts the gateway is deployed and that it did not reject the Actor. +func TestActorEgress(t *testing.T) { + ctx := context.Background() + actorName, _ := createAndResumeActor(t, ctx, "egress", egressTemplate) + router := mustRouterClient(t, ctx) + defer router.Close() + + // The egress demo fetches the URL it is given and echoes the upstream + // status and body back. + payload := []byte(`{"url":"http://example.com/"}`) + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + response, err := router.PostJSON(ctx, actorRef, "/", payload) + if err != nil { + t.Fatalf("POST to egress Actor through ingress: %v", err) + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + t.Fatalf("reading egress response body (HTTP %d): %v", response.StatusCode, err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("Actor egress fetch returned HTTP %d, want 200; body: %s", response.StatusCode, body) + } + t.Logf("Actor egress fetch succeeded; body: %s", body) +} + +func createAndResumeActor(t *testing.T, ctx context.Context, prefix string, template actorTemplate) (string, *ateapipb.Actor) { t.Helper() clients := e2e.GetClients() actorName := fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano()) @@ -80,10 +126,10 @@ func createAndResumeActor(t *testing.T, ctx context.Context, prefix string) (str }) if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: networkingAtespace, Name: actorName}, - ActorTemplateNamespace: "ate-demo-counter", - ActorTemplateName: "counter", + ActorTemplateNamespace: template.namespace, + ActorTemplateName: template.name, }}); err != nil { - t.Fatalf("CreateActor: %v (deploy the fixture with --deploy-demo-counter)", err) + t.Fatalf("CreateActor from %s/%s: %v (deploy the fixture with %s)", template.namespace, template.name, err, template.deployFlag) } t.Cleanup(func() { _, _ = clients.SubstrateAPI.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef}) diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 47d7e6238..6eced4b7a 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -112,7 +112,7 @@ spec: # # SEE(lior): this flag lived on atelet on the pre-rebase branch; PR 708 # moved egress-gateway configuration to ateapi, so it is set here now. - - --egress-gateway-address=ateway-egress.ate-system.svc:443 + - --egress-gateway-address=atenet-egress.ate-system.svc:443 # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s - --drain-timeout=15s diff --git a/manifests/ate-install/ateway-egress.yaml b/manifests/ate-install/atenet-egress.yaml similarity index 86% rename from manifests/ate-install/ateway-egress.yaml rename to manifests/ate-install/atenet-egress.yaml index 37d7f9d7a..924c440aa 100644 --- a/manifests/ate-install/ateway-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -27,7 +27,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: ateway-egress + name: atenet-egress namespace: ate-system --- # RBAC for the co-located ext_proc sidecar (atenet router), which watches @@ -35,7 +35,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: ateway-egress + name: atenet-egress rules: - apiGroups: - "ate.dev" @@ -49,20 +49,20 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: ateway-egress + name: atenet-egress subjects: - kind: ServiceAccount - name: ateway-egress + name: atenet-egress namespace: ate-system roleRef: kind: ClusterRole - name: ateway-egress + name: atenet-egress apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap metadata: - name: ateway-egress + name: atenet-egress namespace: ate-system data: envoy.yaml: | @@ -75,7 +75,11 @@ data: address: socket_address: { address: 0.0.0.0, port_value: 443 } filter_chains: - - transport_socket: + # Named so ext_proc can read it back as xds.filter_chain_name. Must + # match EgressFilterChainName in + # cmd/atenet/internal/router/extproc_egress.go. + - name: egress + transport_socket: name: envoy.transport_sockets.tls typed_config: "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext @@ -130,10 +134,13 @@ data: # certificate, not from request headers. The actor sends no # identity headers any more, and logging attacker-controlled # ones would have made this log lie about who egressed. - # peer= is the leaf subject (CN carries the actor); the - # atespace/UID live in the ActorIdentity extension, which - # Envoy cannot format — the ext_proc handler logs those. - inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer=%DOWNSTREAM_PEER_SUBJECT% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" + # peer_san= is the actor's SPIFFE URI SAN, + # spiffe://substrate-actor.local/atespace//actor/ + # — the only actor-identifying field Envoy can format + # (ateapi mints these with an empty Subject, and the UID + # lives in the ActorIdentity extension Envoy cannot read). + # The ext_proc sidecar logs atespace/actor/UID separately. + inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" route_config: name: connect_route virtual_hosts: @@ -163,13 +170,14 @@ data: failure_mode_allow: false # How the ext_proc server tells egress from ingress. It applies # opposite trust models to the two directions, and dispatches on - # this Envoy-asserted listener name so that no client can select - # the egress path by crafting a request. The value must match - # EgressListenerName in cmd/atenet/internal/router/extproc_egress.go; - # renaming the listener above without updating it fails closed - # (egress requests take the ingress path and 404). + # this Envoy-asserted filter chain name so that no client can + # select the egress path by crafting a request. The value must + # match EgressFilterChainName in + # cmd/atenet/internal/router/extproc_egress.go; renaming the + # filter chain above without updating it fails closed (egress + # requests take the ingress path and 404). request_attributes: - - xds.listener_name + - xds.filter_chain_name processing_mode: request_header_mode: SEND response_header_mode: SKIP @@ -224,21 +232,21 @@ data: apiVersion: apps/v1 kind: Deployment metadata: - name: ateway-egress + name: atenet-egress namespace: ate-system labels: - app: ateway-egress + app: atenet-egress spec: replicas: 1 selector: matchLabels: - app: ateway-egress + app: atenet-egress template: metadata: labels: - app: ateway-egress + app: atenet-egress spec: - serviceAccountName: ateway-egress + serviceAccountName: atenet-egress securityContext: # Allow the non-root envoy user to bind :443. sysctls: @@ -258,9 +266,9 @@ spec: - -c - /etc/envoy/envoy.yaml - --service-node - - ateway-egress + - atenet-egress - --service-cluster - - ateway-egress + - atenet-egress ports: - name: https containerPort: 443 @@ -302,8 +310,9 @@ spec: - --namespace=ate-system - --port-extproc=50051 - --extproc-address=127.0.0.1 - - --ateapi-address=api.ate-system.svc:443 - - --ateapi-auth=mtls + - --ateapi-address=dns:///api.ate-system.svc:443 + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem # Same bundle Envoy validates the handshake against. The handler # re-verifies the chain in Go and reads the ActorIdentity extension out # of it; without this flag the router has no actor-identity roots and @@ -327,13 +336,22 @@ spec: port: extproc periodSeconds: 10 volumeMounts: + # Trust bundle used to verify ateapi's servicedns serving cert. + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + # ext-proc's own client identity presented to ateapi. + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + # Actor-identity CA roots, for --actor-identity-ca-file above. - name: actor-id-ca-certs mountPath: /run/actor-id-ca-certs readOnly: true volumes: - name: config configMap: - name: ateway-egress + name: atenet-egress - name: servicedns projected: sources: @@ -380,12 +398,12 @@ spec: apiVersion: v1 kind: Service metadata: - name: ateway-egress + name: atenet-egress namespace: ate-system spec: type: ClusterIP selector: - app: ateway-egress + app: atenet-egress ports: - name: https port: 443 diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 3e916e80f..188bb8778 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -23,7 +23,7 @@ resources: - ../ate-controller.yaml - ../atelet.yaml - ../atenet-dns.yaml - - ../ateway-egress.yaml + - ../atenet-egress.yaml - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 8be23e96f..c6f9e5e51 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -26,7 +26,7 @@ resources: - ../ate-controller.yaml - ./atelet - ../atenet-dns.yaml - - ../ateway-egress.yaml + - ../atenet-egress.yaml - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml From 17572a6302e0bb51342d438c7df7e6614e5959d0 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Wed, 5 Aug 2026 11:42:24 -0700 Subject: [PATCH 3/6] fixes --- cmd/atenet/internal/router/extproc_egress.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/atenet/internal/router/extproc_egress.go b/cmd/atenet/internal/router/extproc_egress.go index febafaae3..20b18b360 100644 --- a/cmd/atenet/internal/router/extproc_egress.go +++ b/cmd/atenet/internal/router/extproc_egress.go @@ -44,14 +44,11 @@ const ( // filter chain that accepted the request. The egress Envoy asks for it via // request_attributes on its ext_proc filter. // - // SEE(lior): this was xds.listener_name, which reads more naturally but - // which Envoy 1.34 cannot parse: it logs "error parsing cel expression - // xds.listener_name" at trace level, then sends the ProcessingRequest with - // an empty attributes map rather than failing config load. Because an - // absent attribute means "ingress" (the fail-safe direction), every egress - // CONNECT silently took the ingress path and 404'd on the actor DNS name - // parse. xds.filter_chain_name parses on the same Envoy build and is - // equally Envoy-asserted, so the trust model is unchanged. + // Do not "improve" this to xds.listener_name: Envoy 1.34 cannot parse that + // one, and rather than failing config load it logs "error parsing cel + // expression" at trace level and sends an empty attributes map. An absent + // attribute means "ingress" here, so every egress CONNECT would silently + // take the ingress path and 404 on the actor DNS name parse. FilterChainNameAttribute = "xds.filter_chain_name" // forwardedClientCertHeader is the header Envoy fills in with details of From 86d8bcf8dc6b4626fdcabcacabff366f9ade1a3b Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Thu, 6 Aug 2026 14:07:47 -0700 Subject: [PATCH 4/6] apply feedback 08/06 --- cmd/atenet/internal/router/README.md | 43 +++ cmd/atenet/internal/router/cmd.go | 13 +- cmd/atenet/internal/router/config.go | 54 +++- cmd/atenet/internal/router/config_test.go | 94 ++++-- cmd/atenet/internal/router/controller.go | 25 +- cmd/atenet/internal/router/dataplane.go | 10 +- .../{extproc_egress.go => egress/egress.go} | 171 +++++------ .../egress_test.go} | 184 +++--------- cmd/atenet/internal/router/extproc.go | 270 ------------------ .../internal/router/extproc/dispatch.go | 83 ++++++ .../internal/router/extproc/dispatch_test.go | 134 +++++++++ cmd/atenet/internal/router/extproc/errors.go | 82 ++++++ .../internal/router/extproc/errors_test.go | 62 ++++ cmd/atenet/internal/router/extproc/extproc.go | 176 ++++++++++++ .../internal/router/extproc/extproc_test.go | 119 ++++++++ cmd/atenet/internal/router/extproc/handler.go | 74 +++++ .../{extproc_in.go => extproc/metadata.go} | 55 ++-- .../metadata_test.go} | 79 +---- cmd/atenet/internal/router/extproc/metrics.go | 111 +++++++ .../internal/router/extproc/metrics_test.go | 142 +++++++++ cmd/atenet/internal/router/extproc/record.go | 117 ++++++++ cmd/atenet/internal/router/extproc_out.go | 105 ------- cmd/atenet/internal/router/health.go | 8 + cmd/atenet/internal/router/health_test.go | 20 ++ .../internal/router/ingress/actorref_test.go | 83 ++++++ .../internal/router/{ => ingress}/errors.go | 79 +++-- .../router/{ => ingress}/errors_test.go | 78 ++--- cmd/atenet/internal/router/ingress/ingress.go | 211 ++++++++++++++ .../ingress_test.go} | 232 ++++----------- .../internal/router/{ => ingress}/metrics.go | 59 ++-- .../internal/router/{ => ingress}/parking.go | 62 ++-- .../router/{ => ingress}/parking_test.go | 2 +- .../internal/router/{ => ingress}/resumer.go | 13 +- .../router/{ => ingress}/resumer_test.go | 4 +- cmd/atenet/internal/router/router.go | 131 +++++---- cmd/atenet/internal/router/status.go | 149 ++-------- cmd/atenet/internal/router/status_test.go | 12 +- cmd/atenet/internal/router/xds.go | 16 +- cmd/atenet/internal/router/xds_test.go | 6 +- demos/egress/README.md | 14 +- manifests/ate-install/atenet-egress.yaml | 58 ++-- manifests/ate-install/atenet-router.yaml | 6 + .../agentgateway/kustomization.yaml | 8 +- .../kind-token-client/kustomization.yaml | 4 +- .../token-client/kustomization.yaml | 4 +- 45 files changed, 2099 insertions(+), 1363 deletions(-) rename cmd/atenet/internal/router/{extproc_egress.go => egress/egress.go} (70%) rename cmd/atenet/internal/router/{extproc_egress_test.go => egress/egress_test.go} (75%) delete mode 100644 cmd/atenet/internal/router/extproc.go create mode 100644 cmd/atenet/internal/router/extproc/dispatch.go create mode 100644 cmd/atenet/internal/router/extproc/dispatch_test.go create mode 100644 cmd/atenet/internal/router/extproc/errors.go create mode 100644 cmd/atenet/internal/router/extproc/errors_test.go create mode 100644 cmd/atenet/internal/router/extproc/extproc.go create mode 100644 cmd/atenet/internal/router/extproc/extproc_test.go create mode 100644 cmd/atenet/internal/router/extproc/handler.go rename cmd/atenet/internal/router/{extproc_in.go => extproc/metadata.go} (52%) rename cmd/atenet/internal/router/{extproc_in_test.go => extproc/metadata_test.go} (57%) create mode 100644 cmd/atenet/internal/router/extproc/metrics.go create mode 100644 cmd/atenet/internal/router/extproc/metrics_test.go create mode 100644 cmd/atenet/internal/router/extproc/record.go delete mode 100644 cmd/atenet/internal/router/extproc_out.go create mode 100644 cmd/atenet/internal/router/ingress/actorref_test.go rename cmd/atenet/internal/router/{ => ingress}/errors.go (60%) rename cmd/atenet/internal/router/{ => ingress}/errors_test.go (72%) create mode 100644 cmd/atenet/internal/router/ingress/ingress.go rename cmd/atenet/internal/router/{extproc_test.go => ingress/ingress_test.go} (61%) rename cmd/atenet/internal/router/{ => ingress}/metrics.go (58%) rename cmd/atenet/internal/router/{ => ingress}/parking.go (81%) rename cmd/atenet/internal/router/{ => ingress}/parking_test.go (99%) rename cmd/atenet/internal/router/{ => ingress}/resumer.go (96%) rename cmd/atenet/internal/router/{ => ingress}/resumer_test.go (99%) diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 8045f4f23..e2e037acf 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -26,6 +26,49 @@ Router has several responsibilities: derived from the parking budget), then writes a drain-complete marker that releases the Envoy container's `preStop` hook. See `drain.go` and `envoydrain.go`. +* Authenticates actor identity on egress: on every CONNECT, the egress Envoy's + ext_proc handler re-verifies the actor's client certificate against the + actor-identity CA, reads the `ActorIdentity` X.509 extension out of it, and + checks the certified UID against the ATE API. + +## packages + +The ext_proc server handles both traffic directions, and they apply opposite +trust models — egress derives the actor identity from a client certificate +Envoy verified against the actor-identity CA, ingress treats every request +header as unauthenticated client input — so the two are kept in separate +packages that cannot reach into each other: + +* `extproc` — the mux, and nothing else. It terminates the ext_proc stream, + decides which direction a request arrived on, dispatches to the `Handler` + registered for that direction, and records latency and outcome. It also owns + the vocabulary both handlers share (`RequestMetadata`, `Result`, `ReqError`). + It imports neither handler package. +* `ingress` — resume, park, and route to the actor's worker. +* `egress` — certificate-based actor-identity authentication for outbound + CONNECTs. + +Direction is decided by the Envoy filter chain that accepted the request +(`xds.filter_chain_name`), never by anything in the request itself, so a client +cannot pick the egress path by crafting one. `router` itself does the wiring. + +## modes + +One binary serves both directions. `--mode` selects which: + +| `--mode` | ext_proc handlers | xDS server + ActorTemplate controller | Kubernetes access | +| --- | --- | --- | --- | +| `ingress` | ingress | yes | yes | +| `egress` | egress | no | none | +| `all` (default) | both | yes | yes | + +The mux refuses a direction this instance was not started to serve (404) rather +than falling back to the other handler, which would run the request through the +wrong trust model. + +Ingress and egress are deployed separately today — `atenet-router` fronts the +ingress Envoy, `atenet-egress` the egress one — because the two scale +independently, not because they need separate binaries. ## status page diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 374bb11b4..8d12cbbf7 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -20,6 +20,8 @@ import ( "time" "github.com/spf13/cobra" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func NewRouterCmd() *cobra.Command { @@ -39,6 +41,7 @@ func NewRouterCmd() *cobra.Command { }, } + cmd.Flags().StringVar((*string)(&cfg.Mode), "mode", string(ModeAll), fmt.Sprintf("Traffic direction this instance serves: %q (also runs the xDS server and ActorTemplate controller for the ingress Envoy), %q (ext_proc only, needs no Kubernetes access), or %q for both. The ext_proc mux refuses a direction this instance was not started to serve rather than falling back to the other one", ModeIngress, ModeEgress, ModeAll)) cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error") cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().BoolVar(&cfg.Standalone, "standalone", false, "Run in standalone mode, bypassing creation of managed deployment and services in Kubernetes cluster") @@ -71,11 +74,11 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().BoolVar(&cfg.Auth.AteapiUseTokenAuth, "ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") cmd.Flags().StringVar(&cfg.Auth.AteapiTokenFile, "ateapi-token-file", "", "Projected SA token file used as Bearer credential. Required with --ateapi-use-token-auth, ignored otherwise.") cmd.Flags().DurationVar(&cfg.RouteTimeout, "route-timeout", defaultRouteTimeout, "Envoy's end-to-end timeout on the workload route, bounding one request from the ingress listener to the actor's response. Raise it for actors whose turns legitimately run long — a harness relaying an LLM completion holds the request open for the whole generation. This does not cover the resume that may precede the request; see --parked-request-budget") - cmd.Flags().DurationVar(&cfg.ParkedRequest.Budget, "parked-request-budget", defaultParkedRequestBudget, "Maximum time a resume flight keeps a request parked (held and retried) waiting for its actor to become routable; concurrent requests for the same actor share one flight and its budget") - cmd.Flags().IntVar(&cfg.ParkedRequest.Max, "parked-request-max", defaultParkedRequestMax, "Maximum number of requests that may be parked simultaneously; excess requests are shed with 503. 0 disables parking (requests fail fast on worker-pool saturation)") - cmd.Flags().DurationVar(&cfg.ParkedRequest.RetryInterval, "parked-request-retry-interval", defaultParkedRequestRetryInterval, "Delay before a parked request's first resume retry") - cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryFactor, "parked-request-retry-factor", defaultParkedRequestRetryFactor, "Multiplier applied to the retry delay after each attempt; must be >= 1") - cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryJitter, "parked-request-retry-jitter", defaultParkedRequestRetryJitter, "Random fraction in [0, 1) added to each retry delay to de-synchronize parked requests") + cmd.Flags().DurationVar(&cfg.ParkedRequest.Budget, "parked-request-budget", ingress.DefaultParkedRequestBudget, "Maximum time a resume flight keeps a request parked (held and retried) waiting for its actor to become routable; concurrent requests for the same actor share one flight and its budget") + cmd.Flags().IntVar(&cfg.ParkedRequest.Max, "parked-request-max", ingress.DefaultParkedRequestMax, "Maximum number of requests that may be parked simultaneously; excess requests are shed with 503. 0 disables parking (requests fail fast on worker-pool saturation)") + cmd.Flags().DurationVar(&cfg.ParkedRequest.RetryInterval, "parked-request-retry-interval", ingress.DefaultParkedRequestRetryInterval, "Delay before a parked request's first resume retry") + cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryFactor, "parked-request-retry-factor", ingress.DefaultParkedRequestRetryFactor, "Multiplier applied to the retry delay after each attempt; must be >= 1") + cmd.Flags().Float64Var(&cfg.ParkedRequest.RetryJitter, "parked-request-retry-jitter", ingress.DefaultParkedRequestRetryJitter, "Random fraction in [0, 1) added to each retry delay to de-synchronize parked requests") cmd.Flags().IntVar(&cfg.ExtProcMaxRequests, "extproc-max-requests", 0, "Circuit-breaker max_requests for Envoy's ext_proc cluster; 0 (the default) derives it as twice --parked-request-max (minimum 1024). Explicit values must be >= --parked-request-max: every parked request holds one slot for its full wait, and the excess is fast-path headroom") // Graceful shutdown knobs. The router sits behind a Service, so // route-drain window is needed: after SIGTERM the readiness flip diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index d7472e8f6..5004055f8 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -17,6 +17,8 @@ package router import ( "fmt" "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) type atenetRouter string @@ -26,6 +28,40 @@ const ( atenetRouterAgentgateway atenetRouter = "agentgateway" ) +// Mode selects which ext_proc directions an atenet instance serves. One binary +// implements both, as two handlers behind the same ext_proc mux, but a +// deployment usually fronts one Envoy and only needs the matching direction: +// ingress and egress scale independently, so they run as separate Deployments +// (atenet-router and atenet-egress). +// +// The mode is a floor on what an instance will answer, not just a hint. The mux +// refuses a direction it has no handler for rather than falling back to the +// other one, and an egress-only instance also skips the ingress control plane +// (the xDS server and the ActorTemplate controller), which is what lets it run +// without any Kubernetes access at all. +type Mode string + +const ( + // ModeIngress serves actor-addressed traffic entering the mesh, and runs + // the xDS server and ActorTemplate controller that configure its Envoy. + ModeIngress Mode = "ingress" + // ModeEgress serves actor CONNECTs leaving the mesh. Nothing else runs: the + // egress Envoy is statically configured, so there is no xDS server, no + // ActorTemplate controller, and no Kubernetes client. + ModeEgress Mode = "egress" + // ModeAll serves both directions from one instance. This is the default, + // and what a single-gateway or local development setup wants. + ModeAll Mode = "all" +) + +// ServesIngress reports whether this mode answers ingress requests. It also +// gates the ingress control plane: the xDS server that configures the ingress +// Envoy and the ActorTemplate controller that feeds it. +func (m Mode) ServesIngress() bool { return m != ModeEgress } + +// ServesEgress reports whether this mode answers egress CONNECTs. +func (m Mode) ServesEgress() bool { return m != ModeIngress } + // authConfig holds the router's client-auth settings for dialing ateapi. // AteapiCAFile always verifies ateapi's serving cert (the servicedns trust // bundle in-cluster). By default the router presents AteapiClientCertPath @@ -42,6 +78,8 @@ type authConfig struct { // routerConfig holds deployment setup and endpoint options for the router node instance. type routerConfig struct { + // Mode restricts the instance to one traffic direction. Empty means ModeAll. + Mode Mode Standalone bool AtenetRouter string Namespace string @@ -100,7 +138,8 @@ type routerConfig struct { // ParkedRequest configures request parking: hold and retry requests whose // actor cannot be served immediately due to transient worker-pool // saturation, instead of failing fast. A non-positive Max disables parking. - ParkedRequest ParkedRequestConfig + // Ingress-only: egress never resumes an actor. + ParkedRequest ingress.ParkedRequestConfig // ExtProcMaxRequests is the circuit-breaker max_requests Envoy applies to // the ext_proc cluster. Every parked request holds one slot for its entire @@ -163,7 +202,7 @@ const drainTimeoutMargin = 5 * time.Second // route ceiling cannot silently stretch shutdown past the pod's grace period // (see defaultRouteTimeout); operators pair a long route timeout with an // explicit --drain-timeout instead. -func (c routerConfig) drainTimeout(parkCfg ParkedRequestConfig) time.Duration { +func (c routerConfig) drainTimeout(parkCfg ingress.ParkedRequestConfig) time.Duration { if c.DrainTimeout > 0 { return c.DrainTimeout } @@ -178,7 +217,12 @@ func (c routerConfig) validate() error { default: return fmt.Errorf("--atenet-router must be %q or %q, got %q", atenetRouterEnvoy, atenetRouterAgentgateway, c.AtenetRouter) } - if err := c.ParkedRequest.validate(); err != nil { + switch c.Mode { + case "", ModeIngress, ModeEgress, ModeAll: + default: + return fmt.Errorf("--mode must be one of %q, %q, or %q, got %q", ModeIngress, ModeEgress, ModeAll, c.Mode) + } + if err := c.ParkedRequest.Validate(); err != nil { return err } @@ -195,9 +239,9 @@ func (c routerConfig) validate() error { if c.DrainTimeout < 0 { return fmt.Errorf("--drain-timeout must not be negative, got %s (0 derives it from --parked-request-budget)", c.DrainTimeout) } - if c.DrainTimeout > 0 && c.ParkedRequest.enabled() && c.DrainTimeout < c.ParkedRequest.normalized().Budget { + if c.DrainTimeout > 0 && c.ParkedRequest.Enabled() && c.DrainTimeout < c.ParkedRequest.Normalized().Budget { return fmt.Errorf("--drain-timeout (%s) must be >= --parked-request-budget (%s): a drain shorter than the parking budget resets parked requests on shutdown instead of letting them finish", - c.DrainTimeout, c.ParkedRequest.normalized().Budget) + c.DrainTimeout, c.ParkedRequest.Normalized().Budget) } return nil } diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index 2dfeae966..0c4b60ec2 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -18,6 +18,8 @@ import ( "strings" "testing" "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func TestRouterConfigValidate(t *testing.T) { @@ -27,16 +29,16 @@ func TestRouterConfigValidate(t *testing.T) { wantErr string // substring; empty means valid }{ { - name: "atenet-router defaults to envoy", - cfg: routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, + name: "defaults are valid (auto breaker, atenet-router defaults to envoy)", + cfg: routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, }, { name: "atenet-router set to envoy is valid", - cfg: routerConfig{AtenetRouter: string(atenetRouterEnvoy), ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, + cfg: routerConfig{AtenetRouter: string(atenetRouterEnvoy), ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, }, { name: "atenet-router set to agentgateway is valid", - cfg: routerConfig{AtenetRouter: string(atenetRouterAgentgateway), ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, + cfg: routerConfig{AtenetRouter: string(atenetRouterAgentgateway), ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, }, { name: "unknown router rejected", @@ -45,47 +47,64 @@ func TestRouterConfigValidate(t *testing.T) { }, { name: "negative extproc-max-requests rejected", - cfg: routerConfig{ExtProcMaxRequests: -1, ParkedRequest: ParkedRequestConfig{Max: 0}}, + cfg: routerConfig{ExtProcMaxRequests: -1, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, wantErr: "must not be negative", }, { name: "explicit breaker below the lot rejected", - cfg: routerConfig{ExtProcMaxRequests: 512, ParkedRequest: ParkedRequestConfig{Max: 1024}}, + cfg: routerConfig{ExtProcMaxRequests: 512, ParkedRequest: ingress.ParkedRequestConfig{Max: 1024}}, wantErr: "must be >= --parked-request-max", }, { name: "explicit breaker equal to the lot accepted", - cfg: routerConfig{ExtProcMaxRequests: 1024, ParkedRequest: ParkedRequestConfig{Max: 1024}}, + cfg: routerConfig{ExtProcMaxRequests: 1024, ParkedRequest: ingress.ParkedRequestConfig{Max: 1024}}, }, { name: "parking disabled ignores the relation", - cfg: routerConfig{ExtProcMaxRequests: 8, ParkedRequest: ParkedRequestConfig{Max: 0}}, + cfg: routerConfig{ExtProcMaxRequests: 8, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, + }, + { + name: "explicit ingress mode accepted", + cfg: routerConfig{Mode: ModeIngress}, + }, + { + name: "explicit egress mode accepted", + cfg: routerConfig{Mode: ModeEgress}, + }, + { + name: "explicit all mode accepted", + cfg: routerConfig{Mode: ModeAll}, + }, + { + name: "unknown mode rejected", + cfg: routerConfig{Mode: "both"}, + wantErr: `--mode must be one of`, }, { name: "drain-timeout below the parking budget rejected", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 2 * time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 2 * time.Second}, wantErr: "must be >= --parked-request-budget", }, { name: "drain-timeout equal to the parking budget accepted", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 5 * time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 5 * time.Second}, }, { name: "drain-timeout above the parking budget accepted", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 30 * time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}, DrainTimeout: 30 * time.Second}, }, { name: "short drain-timeout with parking disabled accepted", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Max: 0}, DrainTimeout: time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Max: 0}, DrainTimeout: time.Second}, }, { name: "negative drain-timeout rejected", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}, DrainTimeout: -time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, DrainTimeout: -time.Second}, wantErr: "--drain-timeout must not be negative", }, { name: "negative drain-delay rejected", - cfg: routerConfig{ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}, DrainDelay: -time.Second}, + cfg: routerConfig{ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}, DrainDelay: -time.Second}, wantErr: "--drain-delay must not be negative", }, } @@ -124,17 +143,42 @@ func TestRouterConfigAtenetRouter(t *testing.T) { } } +// The empty mode is what a routerConfig built in code (rather than from flags) +// carries, and it must behave as ModeAll so nothing silently stops serving. +func TestModeServes(t *testing.T) { + tests := []struct { + mode Mode + wantIngress bool + wantEgress bool + }{ + {mode: "", wantIngress: true, wantEgress: true}, + {mode: ModeAll, wantIngress: true, wantEgress: true}, + {mode: ModeIngress, wantIngress: true, wantEgress: false}, + {mode: ModeEgress, wantIngress: false, wantEgress: true}, + } + for _, tc := range tests { + t.Run(string(tc.mode), func(t *testing.T) { + if got := tc.mode.ServesIngress(); got != tc.wantIngress { + t.Errorf("ServesIngress() = %v, want %v", got, tc.wantIngress) + } + if got := tc.mode.ServesEgress(); got != tc.wantEgress { + t.Errorf("ServesEgress() = %v, want %v", got, tc.wantEgress) + } + }) + } +} + func TestRouterConfigExtProcMaxRequests(t *testing.T) { tests := []struct { name string cfg routerConfig want int }{ - {"auto derives twice the default lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}}, 2 * defaultParkedRequestMax}, - {"auto scales with a larger lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: 4096}}, 8192}, - {"auto floors at Envoy's default when the lot is small", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: 10}}, extProcMaxRequestsFloor}, - {"auto floors when parking is disabled", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: 0}}, extProcMaxRequestsFloor}, - {"explicit value wins over derivation", routerConfig{ExtProcMaxRequests: 1500, ParkedRequest: ParkedRequestConfig{Max: 1024}}, 1500}, + {"auto derives twice the default lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: ingress.DefaultParkedRequestMax}}, 2 * ingress.DefaultParkedRequestMax}, + {"auto scales with a larger lot", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: 4096}}, 8192}, + {"auto floors at Envoy's default when the lot is small", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: 10}}, extProcMaxRequestsFloor}, + {"auto floors when parking is disabled", routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ingress.ParkedRequestConfig{Max: 0}}, extProcMaxRequestsFloor}, + {"explicit value wins over derivation", routerConfig{ExtProcMaxRequests: 1500, ParkedRequest: ingress.ParkedRequestConfig{Max: 1024}}, 1500}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -149,19 +193,19 @@ func TestRouterConfigDrainTimeout(t *testing.T) { tests := []struct { name string cfg routerConfig - parkCfg ParkedRequestConfig + parkCfg ingress.ParkedRequestConfig want time.Duration }{ { name: "auto derives budget + route timeout + margin", cfg: routerConfig{DrainTimeout: 0}, - parkCfg: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.normalized(), + parkCfg: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.Normalized(), want: 5*time.Second + defaultRouteTimeout + drainTimeoutMargin, }, { name: "auto scales with a larger budget", cfg: routerConfig{DrainTimeout: 0}, - parkCfg: ParkedRequestConfig{Budget: 30 * time.Second, Max: 1024}.normalized(), + parkCfg: ingress.ParkedRequestConfig{Budget: 30 * time.Second, Max: 1024}.Normalized(), want: 30*time.Second + defaultRouteTimeout + drainTimeoutMargin, }, { @@ -170,13 +214,13 @@ func TestRouterConfigDrainTimeout(t *testing.T) { // normalized() fills Budget even when Max disables parking, so the // derived drain still covers a later re-enable without a restart // surprise. - parkCfg: ParkedRequestConfig{Max: 0}.normalized(), - want: defaultParkedRequestBudget + defaultRouteTimeout + drainTimeoutMargin, + parkCfg: ingress.ParkedRequestConfig{Max: 0}.Normalized(), + want: ingress.DefaultParkedRequestBudget + defaultRouteTimeout + drainTimeoutMargin, }, { name: "explicit value wins over derivation", cfg: routerConfig{DrainTimeout: 42 * time.Second}, - parkCfg: ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.normalized(), + parkCfg: ingress.ParkedRequestConfig{Budget: 5 * time.Second, Max: 1024}.Normalized(), want: 42 * time.Second, }, } diff --git a/cmd/atenet/internal/router/controller.go b/cmd/atenet/internal/router/controller.go index e66243d5b..1554e1051 100644 --- a/cmd/atenet/internal/router/controller.go +++ b/cmd/atenet/internal/router/controller.go @@ -23,14 +23,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// Controller monitors ActorTemplates and coordinates configuration updates -// for the Envoy xDS and external processing servers. +// Controller monitors ActorTemplates and coordinates configuration updates for +// the ingress Envoy's xDS server. It is part of the ingress control plane and +// only runs in a mode that serves ingress — the egress Envoy is statically +// configured and has no templates to watch. type Controller struct { - k8sClient client.Client - clientset kubernetes.Interface - cfg routerConfig - xdsSrv *XdsServer - extprocSrv *ExtProcServer + k8sClient client.Client + clientset kubernetes.Interface + cfg routerConfig + xdsSrv *XdsServer atStore atStore envoyRunner *envoyrunner @@ -41,7 +42,6 @@ func NewController( clientset kubernetes.Interface, cfg routerConfig, xdsSrv *XdsServer, - extprocSrv *ExtProcServer, ) *Controller { xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr) @@ -53,11 +53,10 @@ func NewController( } return &Controller{ - k8sClient: k8sClient, - clientset: clientset, - cfg: cfg, - xdsSrv: xdsSrv, - extprocSrv: extprocSrv, + k8sClient: k8sClient, + clientset: clientset, + cfg: cfg, + xdsSrv: xdsSrv, atStore: store, envoyRunner: newEnvoyRunner(k8sClient, cfg), diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index af305e5e0..93bdc5ed5 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -22,6 +22,8 @@ import ( "time" "golang.org/x/sync/errgroup" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) type dataplaneHealthCheck struct { @@ -44,7 +46,7 @@ func (r atenetRouter) healthCheck() dataplaneHealthCheck { } } -func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) error { +func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ingress.ParkedRequestConfig, traceRootSamplingPercent float64) error { switch s.cfg.atenetRouter() { case atenetRouterEnvoy: s.startEnvoyDataplane(ctx, g, parkCfg, traceRootSamplingPercent) @@ -56,7 +58,7 @@ func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, pa return nil } -func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig, traceRootSamplingPercent float64) { +func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ingress.ParkedRequestConfig, traceRootSamplingPercent float64) { xdsSrv := NewXdsServer(s.cfg.XdsPort) xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr) setOtlpCollector(ctx, xdsSrv, s.cfg.OtlpCollectorAddress) @@ -64,7 +66,7 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou xdsSrv.SetRouteTimeout(s.cfg.RouteTimeout) xdsSrv.SetExtProcMaxRequests(s.cfg.extProcMaxRequests()) - if parkCfg.enabled() { + if parkCfg.Enabled() { // Envoy must keep a parked request open at least as long as the router // will hold it; add a margin so the router surfaces its own 503 first. xdsSrv.SetExtProcMessageTimeout(parkCfg.Budget + 5*time.Second) @@ -72,7 +74,7 @@ func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Grou xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath) xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath, s.cfg.UpstreamSpiffePrefix) - ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv, s.extprocSrv) + ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv) // Envoy receives all routing configuration from the local xDS server. g.Go(func() error { diff --git a/cmd/atenet/internal/router/extproc_egress.go b/cmd/atenet/internal/router/egress/egress.go similarity index 70% rename from cmd/atenet/internal/router/extproc_egress.go rename to cmd/atenet/internal/router/egress/egress.go index 20b18b360..2031691fe 100644 --- a/cmd/atenet/internal/router/extproc_egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -12,7 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +// Package egress implements the ext_proc handler for traffic leaving the mesh: +// it authenticates the actor behind an egress CONNECT before the gateway +// tunnels it out. +// +// The identity this handler acts on comes from the actor certificate presented +// in the mTLS handshake and signed by the actor-identity CA — never from a +// request header. That is the opposite of the ingress package's model, where +// every header is unauthenticated client input. Keeping the two in separate +// packages keeps that difference explicit; the ext_proc mux is what guarantees +// a request only ever reaches the handler for the filter chain that accepted +// it. +package egress import ( "context" @@ -30,27 +41,13 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) const ( - // EgressFilterChainName is the Envoy filter chain that terminates actor - // egress CONNECTs. It must stay in sync with the filter chain name in - // manifests/ate-install/atenet-egress.yaml. - EgressFilterChainName = "egress" - // FilterChainNameAttribute is the CEL attribute carrying the name of the - // filter chain that accepted the request. The egress Envoy asks for it via - // request_attributes on its ext_proc filter. - // - // Do not "improve" this to xds.listener_name: Envoy 1.34 cannot parse that - // one, and rather than failing config load it logs "error parsing cel - // expression" at trace level and sends an empty attributes map. An absent - // attribute means "ingress" here, so every egress CONNECT would silently - // take the ingress path and 404 on the actor DNS name parse. - FilterChainNameAttribute = "xds.filter_chain_name" - // forwardedClientCertHeader is the header Envoy fills in with details of // the mTLS peer, including the PEM chain it validated. The egress filter // chain sets forward_client_cert_details: SANITIZE_SET, so whatever a @@ -67,84 +64,62 @@ const ( xfccChainKey = "chain" ) -// isEgressRequest reports whether an ext_proc RequestHeaders callback arrived on -// the egress gateway's filter chain rather than on an ingress one. This lets one -// ext_proc server handle both directions off the same stream. -// -// Dispatch is by filter chain, not by :method, because the two handlers apply -// opposite trust models: on egress the actor identity comes from a client -// certificate Envoy validated, while on ingress every request header is -// unauthenticated client input. Keying on :method would let any external client -// sending CONNECT select the egress handler and use its denial messages as an -// actor-existence and status oracle. Envoy asserts the filter chain name; the -// request cannot influence it. -// -// An unrecognized or absent attribute means ingress, the fail-safe direction: an -// egress request misrouted to the ingress handler fails to parse as an actor DNS -// name and 404s, whereas the reverse leaks control-plane state. -func isEgressRequest(req *extprocv3.ProcessingRequest) bool { - return filterChainName(req) == EgressFilterChainName +// Handler authenticates the actor behind each egress CONNECT. +type Handler struct { + apiClient ateapipb.ControlClient + // actorIdentityRoots is the actor-identity CA bundle every actor + // certificate must chain to. Nil means the gateway cannot authenticate + // anyone, and every CONNECT fails closed. + actorIdentityRoots *x509.CertPool } -// filterChainName returns the xds.filter_chain_name attribute Envoy attached to -// the request, or "" when the listener did not request the attribute. The -// attributes map is keyed by the ext_proc filter's name within the HCM chain, -// which we do not want to hardcode here, so scan every entry. -func filterChainName(req *extprocv3.ProcessingRequest) string { - for _, attrs := range req.GetAttributes() { - if v, ok := attrs.GetFields()[FilterChainNameAttribute]; ok { - return v.GetStringValue() - } - } - return "" +// New builds the egress handler. actorIdentityRoots is the same trust bundle +// the egress listener uses as its trusted_ca; see verifyActorCertificate for +// why the check is made again here. +func New(apiClient ateapipb.ControlClient, actorIdentityRoots *x509.CertPool) *Handler { + return &Handler{apiClient: apiClient, actorIdentityRoots: actorIdentityRoots} } -// handleEgressRequestHeaders authenticates the actor behind an egress CONNECT -// before the gateway tunnels it out, using the actor certificate atunnel -// presented in the mTLS handshake. Nothing the actor can write — no CONNECT -// header, no request metadata — contributes to the identity; the only inputs -// are the certificate the actor-identity CA signed and the control plane's own -// view of that actor. +func (h *Handler) Direction() extproc.Direction { return extproc.DirectionEgress } + +// HandleRequestHeaders authenticates the actor behind an egress CONNECT before +// the gateway tunnels it out, using the actor certificate atunnel presented in +// the mTLS handshake. Nothing the actor can write — no CONNECT header, no +// request metadata — contributes to the identity; the only inputs are the +// certificate the actor-identity CA signed and the control plane's own view of +// that actor. // // Authorization by destination and credential/token injection are deliberately // a TODO once we have SessionIdentity RPC service figured out. // -// The signature mirrors handleRequestHeaders so Process can dispatch to either -// with a single branch. The (target, tmplNs, tmplName) results are unused for -// egress and returned empty. -// -// The trailing ResumeOutcome exists only to match handleRequestHeaders. -// Egress never resumes an actor — it requires one already RUNNING - -// so every path returns ResumeOutcomeNone. -func (s *ExtProcServer) handleEgressRequestHeaders( - ctx context.Context, - reqHeaders *extprocv3.HttpHeaders, -) (*extprocv3.HeadersResponse, *requestMetadata, string, string, string, ResumeOutcome, error) { - metadata := newRequestMetadata(reqHeaders.Headers.GetHeaders()) - - // Dispatch is by filter chain, so reaching here means the egress chain - // accepted the request. That chain only routes CONNECT (its sole route is a - // connect_matcher), so anything else is config drift rather than a client - // the gateway should tunnel for. - if !strings.EqualFold(metadata.method, "CONNECT") { - return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_MethodNotAllowed, - "egress denied: expected CONNECT, got %q", metadata.method) +// Egress never resumes an actor — it requires one already RUNNING — and picks +// no upstream of its own, so the Result carries neither a resume outcome nor a +// target. +func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestMetadata) (extproc.Result, error) { + // Dispatch is by filter chain, so reaching here means the egress listener + // accepted the request. That listener only routes CONNECT (its sole route + // is a connect_matcher), so anything else is a config drift rather than a + // client the gateway should tunnel for. + if !strings.EqualFold(md.Method, "CONNECT") { + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_MethodNotAllowed, + "egress denied: expected CONNECT, got %q", md.Method) } // No roots means the gateway cannot authenticate anyone. Fail closed, and // as 503 rather than 403: this is our misconfiguration, not the actor's. - if s.actorIdentityRoots == nil { - return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_ServiceUnavailable, + if h.actorIdentityRoots == nil { + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_ServiceUnavailable, "egress unavailable: no actor-identity CA configured") } - identity, err := s.authenticateActorCertificate(metadata) + identity, err := h.authenticateActorCertificate(md) if err != nil { - // The message stays generic on purpose: an actor that fails - // authentication has not proven it is anyone, so it gets no detail - // about why. The specific reason is logged below instead. + // The body stays generic on purpose: an actor that fails authentication + // has not proven it is anyone, so it gets no detail about why. The + // specific reason rides along as the wrapped cause, which only the + // server-side log below reads. slog.WarnContext(ctx, "egress denied: actor certificate rejected", slog.Any("err", err)) - return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + return extproc.Result{}, extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, "egress denied: invalid actor certificate") } @@ -152,22 +127,22 @@ func (s *ExtProcServer) handleEgressRequestHeaders( actorName := identity.ActorName actorUID := identity.ActorUid // For a CONNECT the :authority is the actor's original destination (IP:port). - destination := metadata.host + destination := md.Host // The CA only ever mints these from control-plane state, so a name that is // not a legal resource name means the CA or its inputs are compromised. if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) { - return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, "egress denied: invalid actor identity %q/%q", atespace, actorName) } // Confirm the certified actor still exists. The name is only a lookup key // here; the UID below is what actually authorizes. - actor, err := s.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{ + actor, err := h.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: actorName}, }) if err != nil { - return nil, metadata, "", "", "", ResumeOutcomeNone, mapEgressIdentityError(atespace, actorName, err) + return extproc.Result{}, mapEgressIdentityError(atespace, actorName, err) } // Authorize on the UID, not the name. Names are reused: delete an actor and @@ -181,13 +156,13 @@ func (s *ExtProcServer) handleEgressRequestHeaders( slog.String("actor", actorName), slog.String("certificateActorUid", actorUID), slog.String("currentActorUid", uid)) - return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, "egress denied: actor %q/%q is not the actor this certificate was issued to", atespace, actorName) } // The actor performing egress must actually be running. if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING { - return nil, metadata, "", "", "", ResumeOutcomeNone, newReqError(envoy_type.StatusCode_Forbidden, + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, "egress denied: actor %q/%q is %s, not running", atespace, actorName, actor.GetStatus()) } @@ -201,16 +176,18 @@ func (s *ExtProcServer) handleEgressRequestHeaders( // Identity is authenticated; let the CONNECT proceed unchanged. Milestone 2 // would additionally authorize `destination` and inject upstream credentials // here by returning a HeaderMutation. - return &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{}, - }, metadata, "", "", "", ResumeOutcomeNone, nil + return extproc.Result{ + Response: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, nil } // authenticateActorCertificate turns the mTLS peer certificate Envoy recorded // on the request into a verified ActorIdentity, or an error describing why it // cannot be trusted. -func (s *ExtProcServer) authenticateActorCertificate(metadata *requestMetadata) (*substratex509.ActorIdentity, error) { - header := metadata.headers[forwardedClientCertHeader] +func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*substratex509.ActorIdentity, error) { + header := md.Header(forwardedClientCertHeader) if header == "" { return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader) } @@ -218,7 +195,7 @@ func (s *ExtProcServer) authenticateActorCertificate(metadata *requestMetadata) if err != nil { return nil, err } - return s.verifyActorCertificate(chain) + return h.verifyActorCertificate(chain) } // verifyActorCertificate checks that chain[0] is a live, non-CA, client-auth @@ -250,7 +227,7 @@ func (s *ExtProcServer) authenticateActorCertificate(metadata *requestMetadata) // this function is the thing to delete — but keep the ActorIdentity extraction // and the IsCA/EKU/purpose checks below it, because Envoy does none of those. // ########################################################################### -func (s *ExtProcServer) verifyActorCertificate(chain []*x509.Certificate) (*substratex509.ActorIdentity, error) { +func (h *Handler) verifyActorCertificate(chain []*x509.Certificate) (*substratex509.ActorIdentity, error) { leaf := chain[0] intermediates := x509.NewCertPool() for _, cert := range chain[1:] { @@ -275,7 +252,7 @@ func (s *ExtProcServer) verifyActorCertificate(chain []*x509.Certificate) (*subs return nil, fmt.Errorf("actor certificate cannot authenticate a TLS client") } if _, err := leaf.Verify(x509.VerifyOptions{ - Roots: s.actorIdentityRoots, + Roots: h.actorIdentityRoots, Intermediates: intermediates, CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, @@ -430,26 +407,26 @@ func unquoteXFCC(value string) string { } // mapEgressIdentityError converts a GetActor failure into a client-facing -// ext_proc denial. An unknown actor is treated as a forbidden (the actor was +// ext_proc denial. An unknown actor is treated as forbidden (the actor was // deleted out from under a still-valid certificate); transient control-plane // failures fail closed with 503. func mapEgressIdentityError(atespace, actorName string, err error) error { switch status.Code(err) { case codes.NotFound: - return newReqError(envoy_type.StatusCode_Forbidden, + return extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, "egress denied: unknown actor %q/%q", atespace, actorName) case codes.Unavailable, codes.DeadlineExceeded: - return newReqError(envoy_type.StatusCode_ServiceUnavailable, + return extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, err, "egress identity check unavailable for %q/%q: %v", atespace, actorName, err) default: - return newReqError(envoy_type.StatusCode_Forbidden, + return extproc.WrapReqError(envoy_type.StatusCode_Forbidden, err, "egress denied for %q/%q: %v", atespace, actorName, err) } } -// loadActorIdentityRoots reads the actor-identity CA trust bundle the egress +// LoadActorIdentityRoots reads the actor-identity CA trust bundle the egress // gateway verifies actor client certificates against. -func loadActorIdentityRoots(pemBytes []byte) (*x509.CertPool, error) { +func LoadActorIdentityRoots(pemBytes []byte) (*x509.CertPool, error) { roots := x509.NewCertPool() if !roots.AppendCertsFromPEM(pemBytes) { return nil, fmt.Errorf("actor-identity CA bundle contains no certificates") diff --git a/cmd/atenet/internal/router/extproc_egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go similarity index 75% rename from cmd/atenet/internal/router/extproc_egress_test.go rename to cmd/atenet/internal/router/egress/egress_test.go index 41b193380..1939cd99a 100644 --- a/cmd/atenet/internal/router/extproc_egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package egress import ( "context" @@ -33,121 +33,16 @@ import ( "time" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/structpb" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/substratex509" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) -// connectRequest builds a RequestHeaders ProcessingRequest for an egress -// CONNECT, optionally attributed to a filter chain. filterKey is the ext_proc -// filter name Envoy keys the attributes map by. -func connectRequest(filterKey, chain string) *extprocv3.ProcessingRequest { - req := &extprocv3.ProcessingRequest{ - Request: &extprocv3.ProcessingRequest_RequestHeaders{ - RequestHeaders: &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":method", RawValue: []byte("CONNECT")}, - {Key: ":authority", RawValue: []byte("10.0.0.9:443")}, - }, - }, - }, - }, - } - if chain != "" { - req.Attributes = map[string]*structpb.Struct{ - filterKey: { - Fields: map[string]*structpb.Value{ - FilterChainNameAttribute: structpb.NewStringValue(chain), - }, - }, - } - } - return req -} - -func TestIsEgressRequest(t *testing.T) { - tests := []struct { - name string - filterKey string - chain string - want bool - }{ - { - name: "egress filter chain", - filterKey: "envoy.filters.http.ext_proc", - chain: EgressFilterChainName, - want: true, - }, - { - name: "egress filter chain under a renamed filter", - filterKey: "some.custom.ext_proc.name", - chain: EgressFilterChainName, - want: true, - }, - { - // The pre-listener-dispatch hole: an external client sending - // CONNECT to the ingress gateway must not reach the egress handler, - // whose denials would otherwise report whether an arbitrary actor - // exists and is running. - name: "CONNECT on the ingress HTTP listener", - filterKey: "envoy.filters.http.ext_proc", - chain: IngressHTTPListener, - want: false, - }, - { - name: "CONNECT on the ingress HTTPS listener", - filterKey: "envoy.filters.http.ext_proc", - chain: IngressHTTPSListener, - want: false, - }, - { - // A listener that never requested the attribute falls back to - // ingress, the fail-safe direction. - name: "no attributes at all", - chain: "", - want: false, - }, - { - name: "unrecognised filter chain", - filterKey: "envoy.filters.http.ext_proc", - chain: "some-other-chain", - want: false, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := isEgressRequest(connectRequest(tc.filterKey, tc.chain)); got != tc.want { - t.Errorf("isEgressRequest() = %v, want %v", got, tc.want) - } - }) - } -} - -// A request the client dresses up to look like egress must not be enough: only -// the Envoy-asserted filter chain name selects the egress handler. -func TestIsEgressRequestIgnoresClientSuppliedAttributeHeader(t *testing.T) { - req := connectRequest("envoy.filters.http.ext_proc", IngressHTTPListener) - rh := req.GetRequestHeaders().GetHeaders() - rh.Headers = append(rh.Headers, - &corev3.HeaderValue{Key: FilterChainNameAttribute, RawValue: []byte(EgressFilterChainName)}, - &corev3.HeaderValue{Key: "x-envoy-filter-chain-name", RawValue: []byte(EgressFilterChainName)}, - ) - - if isEgressRequest(req) { - t.Error("isEgressRequest() = true for a client-forged filter chain header, want false") - } -} - -// --- actor certificate authentication ------------------------------------- - const ( testEgressAtespace = "default" testEgressActor = "my-actor" @@ -294,10 +189,9 @@ func xfccHeaderDER(chain ...[]byte) string { url.PathEscape(buf.String())) } -// egressServer builds an ExtProcServer whose GetActor returns actor/err. -func egressServer(roots *x509.CertPool, actor *ateapipb.Actor, err error) *ExtProcServer { - client := &egressMockClient{actor: actor, err: err} - return NewExtProcServer(50051, client, nil, ParkedRequestConfig{}, nil, false, roots) +// egressHandler builds a Handler whose GetActor returns actor/err. +func egressHandler(roots *x509.CertPool, actor *ateapipb.Actor, err error) *Handler { + return New(&egressMockClient{actor: actor, err: err}, roots) } type egressMockClient struct { @@ -324,8 +218,8 @@ func runningActor() *ateapipb.Actor { } } -// egressHeaders builds the CONNECT the egress listener hands to ext_proc. -func egressHeaders(xfcc string) *extprocv3.HttpHeaders { +// egressMetadata builds the CONNECT the egress listener hands to ext_proc. +func egressMetadata(xfcc string) *extproc.RequestMetadata { headers := []*corev3.HeaderValue{ {Key: ":method", RawValue: []byte("CONNECT")}, {Key: ":authority", RawValue: []byte("93.184.216.34:80")}, @@ -333,7 +227,7 @@ func egressHeaders(xfcc string) *extprocv3.HttpHeaders { if xfcc != "" { headers = append(headers, &corev3.HeaderValue{Key: forwardedClientCertHeader, RawValue: []byte(xfcc)}) } - return &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{Headers: headers}} + return extproc.NewRequestMetadata(headers) } func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) { @@ -341,35 +235,39 @@ func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) { if err == nil { t.Fatalf("expected a denial with status %d, got none", want) } - var re *reqError + var re *extproc.ReqError if !errors.As(err, &re) { - t.Fatalf("error %v is not a *reqError", err) + t.Fatalf("error %v is not a *extproc.ReqError", err) } - if re.statusCode != int(want) { - t.Errorf("status = %d, want %d (%v)", re.statusCode, want, err) + if re.StatusCode != int(want) { + t.Errorf("status = %d, want %d (%v)", re.StatusCode, want, err) } } -func TestHandleEgressRequestHeadersAllowsVerifiedActor(t *testing.T) { +func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, actorCertOptions{}) - s := egressServer(ca.roots(), runningActor(), nil) + h := egressHandler(ca.roots(), runningActor(), nil) - resp, _, _, _, _, resume, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf))) + res, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) if err != nil { - t.Fatalf("handleEgressRequestHeaders() error = %v, want nil", err) + t.Fatalf("HandleRequestHeaders() error = %v, want nil", err) + } + if res.Response == nil { + t.Fatal("HandleRequestHeaders() returned no response") } - if resp == nil { - t.Fatal("handleEgressRequestHeaders() returned no response") + // Egress neither resumes an actor nor picks an upstream. + if res.Resume != "" { + t.Errorf("resume outcome = %q, want %q", res.Resume, "") } - if resume != ResumeOutcomeNone { - t.Errorf("resume outcome = %q, want %q", resume, ResumeOutcomeNone) + if res.Target != "" { + t.Errorf("target = %q, want %q", res.Target, "") } } // Every way an actor certificate can fail to prove an identity has to end in a // denial, never in a tunnel. -func TestHandleEgressRequestHeadersRejectsBadCertificates(t *testing.T) { +func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") otherCA := newTestCA(t, "some-other-ca") @@ -554,8 +452,8 @@ func TestHandleEgressRequestHeadersRejectsBadCertificates(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - s := egressServer(ca.roots(), runningActor(), nil) - _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(tc.xfcc(t))) + h := egressHandler(ca.roots(), runningActor(), nil) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(tc.xfcc(t))) wantStatus(t, err, tc.want) }) } @@ -563,7 +461,7 @@ func TestHandleEgressRequestHeadersRejectsBadCertificates(t *testing.T) { // The certificate authenticates; these cover what the control plane says about // the actor it names. -func TestHandleEgressRequestHeadersAuthorization(t *testing.T) { +func TestHandleRequestHeadersAuthorization(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") tests := []struct { @@ -621,9 +519,9 @@ func TestHandleEgressRequestHeadersAuthorization(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - s := egressServer(ca.roots(), tc.actor, tc.err) + h := egressHandler(ca.roots(), tc.actor, tc.err) leaf := ca.issueActorCert(t, actorCertOptions{}) - _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf))) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) wantStatus(t, err, tc.want) }) } @@ -631,27 +529,25 @@ func TestHandleEgressRequestHeadersAuthorization(t *testing.T) { // An ingress-only router has no actor-identity CA. If an egress CONNECT somehow // reaches it, it must fail closed rather than tunnel unauthenticated traffic. -func TestHandleEgressRequestHeadersWithoutConfiguredCA(t *testing.T) { +func TestHandleRequestHeadersWithoutConfiguredCA(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, actorCertOptions{}) - s := egressServer(nil, runningActor(), nil) + h := egressHandler(nil, runningActor(), nil) - _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), egressHeaders(xfccHeader(leaf))) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) wantStatus(t, err, envoy_type.StatusCode_ServiceUnavailable) } -func TestHandleEgressRequestHeadersRejectsNonConnect(t *testing.T) { +func TestHandleRequestHeadersRejectsNonConnect(t *testing.T) { ca := newTestCA(t, "actor-identity-ca") leaf := ca.issueActorCert(t, actorCertOptions{}) - s := egressServer(ca.roots(), runningActor(), nil) + h := egressHandler(ca.roots(), runningActor(), nil) - headers := egressHeaders(xfccHeader(leaf)) - for _, h := range headers.Headers.Headers { - if h.Key == ":method" { - h.RawValue = []byte("GET") - } - } - _, _, _, _, _, _, err := s.handleEgressRequestHeaders(context.Background(), headers) + md := egressMetadata(xfccHeader(leaf)) + md.Method = "GET" + md.Headers[":method"] = "GET" + + _, err := h.HandleRequestHeaders(context.Background(), md) wantStatus(t, err, envoy_type.StatusCode_MethodNotAllowed) } diff --git a/cmd/atenet/internal/router/extproc.go b/cmd/atenet/internal/router/extproc.go deleted file mode 100644 index 8e9944f5b..000000000 --- a/cmd/atenet/internal/router/extproc.go +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "context" - "crypto/x509" - "errors" - "fmt" - "io" - "log/slog" - "net" - "time" - - "github.com/agent-substrate/substrate/internal/ateattr" - extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" - envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/propagation" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" -) - -// ExtProcServer implements the Envoy external processing gRPC server -// to dynamically manage actor activations based on request traffic. -type ExtProcServer struct { - port int - apiClient ateapipb.ControlClient - recorder *QueryRecorder - resumer *ActorResumer - routeDuration metric.Float64Histogram - parking *parkingLot - routeViaAuthority bool - // actorIdentityRoots are the actor-identity CA certificates the egress - // handler verifies actor client certificates against. Only the ext_proc - // sidecar of the egress gateway sets it; nil makes every egress CONNECT - // fail closed and is the correct state for an ingress-only router. - actorIdentityRoots *x509.CertPool -} - -func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool, actorIdentityRoots *x509.CertPool) *ExtProcServer { - return &ExtProcServer{ - port: port, - apiClient: apiClient, - recorder: NewQueryRecorder(100), - resumer: NewActorResumer(apiClient, withParking(parkCfg)), - routeDuration: routeDuration, - parking: newParkingLot(parkCfg, parkMetrics), - routeViaAuthority: routeViaAuthority, - actorIdentityRoots: actorIdentityRoots, - } -} - -// NewGRPCServer builds the gRPC server with the ext_proc service registered. -// The caller owns its lifecycle: Run serves it, and the drain sequence in -// drain.go stops it — gracefully first so in-flight streams (parked requests -// above all) finish, forcefully past the drain timeout. -func (s *ExtProcServer) NewGRPCServer() *grpc.Server { - grpcServer := grpc.NewServer( - grpc.StatsHandler(otelgrpc.NewServerHandler()), - ) - extprocv3.RegisterExternalProcessorServer(grpcServer, s) - return grpcServer -} - -func (s *ExtProcServer) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { - for { - req, err := stream.Recv() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - - resp := &extprocv3.ProcessingResponse{} - - switch reqType := req.Request.(type) { - case *extprocv3.ProcessingRequest_RequestHeaders: - start := time.Now() - // One ext_proc server handles both directions: actor egress - // CONNECT requests and ingress requests. Which one is decided by - // the accepting listener, not by anything in the request itself - // (see isEgressRequest). - // - handle := s.handleRequestHeaders - if isEgressRequest(req) { - handle = s.handleEgressRequestHeaders - } - hResponse, rqm, target, tmplNs, tmplName, resumeOutcome, err := handle(stream.Context(), reqType.RequestHeaders) - elapsed := time.Since(start) - outcomeStr := classifyOutcome(err) - resumeStr := string(resumeOutcome) - if err != nil { - slog.ErrorContext(stream.Context(), "Error during ext_proc RequestHeaders processing", slog.String("err", err.Error())) - var reqErr *reqError - if errors.As(err, &reqErr) { - resp = immediateResponse(envoy_type.StatusCode(reqErr.statusCode), reqErr.Error()) - } else { - resp = immediateResponse(envoy_type.StatusCode_InternalServerError, err.Error()) - } - s.recordRouteDuration(stream.Context(), elapsed, tmplNs, tmplName, outcomeStr, resumeStr) - s.recorder.AddRouterRequest(start, elapsed, "Error", "-", rqm) - } else { - resp.Response = &extprocv3.ProcessingResponse_RequestHeaders{RequestHeaders: hResponse} - s.recordRouteDuration(stream.Context(), elapsed, tmplNs, tmplName, outcomeStr, resumeStr) - s.recorder.AddRouterRequest(start, elapsed, "Route ok", target, rqm) - } - - default: - // No modification for other processing states, but log because this should - // not be called. - slog.Error("Unexpected request type", slog.String("reqType", fmt.Sprintf("%T", reqType))) - resp.Response = &extprocv3.ProcessingResponse_RequestHeaders{ - RequestHeaders: &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{}, - }, - } - } - - if err := stream.Send(resp); err != nil { - return err - } - } -} - -func (s *ExtProcServer) handleRequestHeaders( - ctx context.Context, - reqHeaders *extprocv3.HttpHeaders, -) (*extprocv3.HeadersResponse, *requestMetadata, string, string, string, ResumeOutcome, error) { - metadata := newRequestMetadata(reqHeaders.Headers.GetHeaders()) - slog.InfoContext(ctx, "Request", slog.String("host", metadata.host)) - - // Envoy doesn't propagate trace context into the ext_proc gRPC - // stream's metadata — the per-request traceparent arrives in the - // HTTP headers carried inside the ProcessingRequest payload. Extract - // from there so our span links to the Envoy ingress span. - ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(metadata.headers)) - ctx, span := otel.Tracer(routerServiceName).Start(ctx, "ExtProc.RequestHeaders") - defer span.End() - - actorRef, err := parseActorRef(metadata.host) - if err != nil { - // Host is invalid, respond with 404. - return nil, metadata, "", "", "", ResumeOutcomeNone, invalidHostErr(metadata.host, err) - } - - // Admit the request to the parking lot before resuming. While resume is - // in-flight the request occupies a slot; if the actor's worker pool is - // momentarily saturated the resumer parks (retries) here rather than failing - // fast. A full lot sheds the request immediately so the router applies - // backpressure instead of queueing without bound. - release, ok := s.parking.enter(ctx) - if !ok { - return nil, metadata, "", "", "", ResumeOutcomeNone, parkingFullErr(actorRef.String()) - } - - slog.InfoContext(ctx, "ResumeActor", slog.Any("actor", actorRef)) - actor, resumeOutcome, err := s.resumer.ResumeActor(ctx, actorRef) - release(parkOutcomeFor(err)) - if err != nil { - return nil, metadata, "", "", "", resumeOutcome, mapResumeError(actorRef, err) - } - - // Actor template identity, used as low-cardinality route-latency metric - // attributes (see recordRouteDuration). - tmplNs := actor.GetActorTemplateNamespace() - tmplName := actor.GetActorTemplateName() - - workerIP := actor.GetWorkerAssignment().GetWorkerPodIp() - slog.InfoContext(ctx, "ResumeActor result", - slog.Any("actor", actorRef), - slog.String("status", actor.GetStatus().String()), - slog.String("workerIP", workerIP)) - - if ip := net.ParseIP(workerIP); ip == nil { - return nil, metadata, "", tmplNs, tmplName, resumeOutcome, newReqError(envoy_type.StatusCode_InternalServerError, - "actor %s routing failed", actorRef) - } - - // The actor is reached through the in-worker atunnel ingress server, which - // listens on :443 (mTLS) and forwards to the actor's :80. The worker no - // longer DNATs pod-IP:80 to the actor, so the router dials :443 and the - // ORIGINAL_DST cluster's upstream TLS context presents the router's - // podidentity client cert (see buildOriginalDstCluster and - // buildUpstreamTransportSocket). - // TODO(bowei) -- handle more than port 80 on the actor. - targetAddr := net.JoinHostPort(workerIP, "443") - - slog.InfoContext(ctx, "Route ok", slog.Any("actor", actorRef), slog.String("targetAddr", targetAddr)) - - // Route by telling the ORIGINAL_DST cluster which worker atunnel address to - // dial, without touching :authority — atunnel authorizes the actor by the - // original Host (actor DNS name). - mutation := &extprocv3.HeaderMutation{} - addRoutingMutations(targetAddr, metadata.host, s.routeViaAuthority, mutation) - - return &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{ - HeaderMutation: mutation, - }, - }, metadata, targetAddr, tmplNs, tmplName, resumeOutcome, nil -} - -func (s *ExtProcServer) recordRouteDuration(ctx context.Context, d time.Duration, tmplNs, tmplName, outcome, resume string) { - if s.routeDuration == nil { - return - } - s.routeDuration.Record(ctx, d.Seconds(), metric.WithAttributes( - ateattr.TemplateNamespaceKey.String(tmplNs), - ateattr.TemplateNameKey.String(tmplName), - ateattr.RouterOutcomeKey.String(outcome), - ateattr.RouterResumeKey.String(resume), - )) -} - -func classifyOutcome(err error) string { - if err == nil { - return "ok" - } - if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { - return "cancelled" - } - if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { - return "timeout" - } - switch status.Code(err) { - case codes.FailedPrecondition: - return "no_capacity" - case codes.Aborted: - return "lock_conflict" - case codes.NotFound: - return "not_found" - case codes.Unavailable: - return "unavailable" - case codes.ResourceExhausted: - return "rate_limited" - } - var re *reqError - if errors.As(err, &re) { - switch envoy_type.StatusCode(re.statusCode) { - case envoy_type.StatusCode_NotFound: - return "not_found" - case envoy_type.StatusCode_ServiceUnavailable: - return "no_capacity" - case envoy_type.StatusCode_GatewayTimeout: - return "timeout" - case envoy_type.StatusCode_TooManyRequests: - return "rate_limited" - } - } - return "resume_error" -} diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go new file mode 100644 index 000000000..38fe10aa7 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" +) + +// Direction is the side of the mesh a request arrived on. It selects the +// handler, so it must come from something Envoy asserts rather than from the +// request. +type Direction string + +const ( + // DirectionIngress is traffic entering the mesh, addressed to an actor. + DirectionIngress Direction = "ingress" + // DirectionEgress is traffic leaving the mesh, tunneled out of an actor. + DirectionEgress Direction = "egress" +) + +const ( + // EgressFilterChainName is the Envoy filter chain that terminates actor + // egress CONNECTs, and so the one that selects the egress handler. It must + // stay in sync with the filter chain name in + // manifests/ate-install/atenet-egress.yaml. + EgressFilterChainName = "egress" + // FilterChainNameAttribute is the CEL attribute carrying the name of the + // filter chain that accepted the request. The egress Envoy asks for it via + // request_attributes on its ext_proc filter. + // + // Do not "improve" this to xds.listener_name: Envoy 1.34 cannot parse that + // one, and rather than failing config load it logs "error parsing cel + // expression" at trace level and sends an empty attributes map. An absent + // attribute means ingress here, so every egress CONNECT would silently take + // the ingress path and 404 on the actor DNS name parse. + FilterChainNameAttribute = "xds.filter_chain_name" +) + +// directionOf reports which direction's handler an ext_proc RequestHeaders +// callback belongs to. +// +// Dispatch is by filter chain, not by :method, because the two directions apply +// opposite trust models: on egress the actor identity comes from a client +// certificate Envoy validated against the actor-identity CA, while on ingress +// every request header is unauthenticated client input. Keying on :method would +// let any external client sending CONNECT select the egress handler and use its +// denial messages as an actor-existence and status oracle. Envoy asserts the +// filter chain name; the request cannot influence it. +// +// An unrecognized or absent attribute means ingress, the fail-safe direction: +// an egress request misrouted to the ingress handler fails to parse as an actor +// DNS name and 404s, whereas the reverse leaks control-plane state. +func directionOf(req *extprocv3.ProcessingRequest) Direction { + if filterChainName(req) == EgressFilterChainName { + return DirectionEgress + } + return DirectionIngress +} + +// filterChainName returns the xds.filter_chain_name attribute Envoy attached to +// the request, or "" when the listener did not request the attribute. The +// attributes map is keyed by the ext_proc filter's name within the HCM chain, +// which we do not want to hardcode here, so scan every entry. +func filterChainName(req *extprocv3.ProcessingRequest) string { + for _, attrs := range req.GetAttributes() { + if v, ok := attrs.GetFields()[FilterChainNameAttribute]; ok { + return v.GetStringValue() + } + } + return "" +} diff --git a/cmd/atenet/internal/router/extproc/dispatch_test.go b/cmd/atenet/internal/router/extproc/dispatch_test.go new file mode 100644 index 000000000..83252eb6d --- /dev/null +++ b/cmd/atenet/internal/router/extproc/dispatch_test.go @@ -0,0 +1,134 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "google.golang.org/protobuf/types/known/structpb" +) + +// connectRequest builds a RequestHeaders ProcessingRequest for a CONNECT, +// optionally attributed to a filter chain. filterKey is the ext_proc filter +// name Envoy keys the attributes map by. +func connectRequest(filterKey, chain string) *extprocv3.ProcessingRequest { + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{ + Headers: []*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("CONNECT")}, + {Key: ":authority", RawValue: []byte("10.0.0.9:443")}, + }, + }, + }, + }, + } + if chain != "" { + req.Attributes = map[string]*structpb.Struct{ + filterKey: { + Fields: map[string]*structpb.Value{ + FilterChainNameAttribute: structpb.NewStringValue(chain), + }, + }, + } + } + return req +} + +// The ingress listener names the router's xDS server assigns. Spelled out here +// rather than imported: the point of this package is that the mux knows nothing +// about either direction's configuration, so the test may not reach for it +// either. +const ( + ingressHTTPListener = "ingress_http_listener" + ingressHTTPSListener = "ingress_https_listener" +) + +func TestDirectionOf(t *testing.T) { + tests := []struct { + name string + filterKey string + chain string + want Direction + }{ + { + name: "egress filter chain", + filterKey: "envoy.filters.http.ext_proc", + chain: EgressFilterChainName, + want: DirectionEgress, + }, + { + name: "egress filter chain under a renamed filter", + filterKey: "some.custom.ext_proc.name", + chain: EgressFilterChainName, + want: DirectionEgress, + }, + { + // The pre-listener-dispatch hole: an external client sending + // CONNECT to the ingress gateway must not reach the egress handler, + // whose denials would otherwise report whether an arbitrary actor + // exists and is running. + name: "CONNECT on the ingress HTTP listener", + filterKey: "envoy.filters.http.ext_proc", + chain: ingressHTTPListener, + want: DirectionIngress, + }, + { + name: "CONNECT on the ingress HTTPS listener", + filterKey: "envoy.filters.http.ext_proc", + chain: ingressHTTPSListener, + want: DirectionIngress, + }, + { + // A listener that never requested the attribute falls back to + // ingress, the fail-safe direction. + name: "no attributes at all", + chain: "", + want: DirectionIngress, + }, + { + name: "unrecognised filter chain", + filterKey: "envoy.filters.http.ext_proc", + chain: "some-other-chain", + want: DirectionIngress, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := directionOf(connectRequest(tc.filterKey, tc.chain)); got != tc.want { + t.Errorf("directionOf() = %v, want %v", got, tc.want) + } + }) + } +} + +// A request the client dresses up to look like egress must not be enough: only +// the Envoy-asserted filter chain name selects the egress handler. +func TestDirectionOfIgnoresClientSuppliedAttributeHeader(t *testing.T) { + req := connectRequest("envoy.filters.http.ext_proc", ingressHTTPListener) + rh := req.GetRequestHeaders().GetHeaders() + rh.Headers = append(rh.Headers, + &corev3.HeaderValue{Key: FilterChainNameAttribute, RawValue: []byte(EgressFilterChainName)}, + &corev3.HeaderValue{Key: "x-envoy-filter-chain-name", RawValue: []byte(EgressFilterChainName)}, + ) + + if got := directionOf(req); got != DirectionIngress { + t.Errorf("directionOf() = %v for a client-forged filter chain header, want %v", got, DirectionIngress) + } +} diff --git a/cmd/atenet/internal/router/extproc/errors.go b/cmd/atenet/internal/router/extproc/errors.go new file mode 100644 index 000000000..12b1f0672 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/errors.go @@ -0,0 +1,82 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "fmt" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +// ReqError is a handler's denial of a request: an HTTP-mappable status code +// and a client-safe message the mux turns into an immediate response. The +// underlying cause (if any) is preserved via Unwrap so logs can inspect the +// full chain without leaking server-side detail into the response body. +type ReqError struct { + Msg string + Cause error + StatusCode int +} + +func (e *ReqError) Error() string { return e.Msg } +func (e *ReqError) Unwrap() error { return e.Cause } + +// NewReqError builds a ReqError whose body is the formatted message and no +// wrapped cause. Use WrapReqError when a cause is available. +func NewReqError(code envoy_type.StatusCode, format string, args ...any) error { + return &ReqError{ + Msg: fmt.Sprintf(format, args...), + StatusCode: int(code), + } +} + +// WrapReqError builds a ReqError that keeps cause reachable through Unwrap +// while answering the client with only the formatted message. +func WrapReqError(code envoy_type.StatusCode, cause error, format string, args ...any) error { + return &ReqError{ + Msg: fmt.Sprintf(format, args...), + Cause: cause, + StatusCode: int(code), + } +} + +// ImmediateResponse tells Envoy to answer the request itself, without going +// upstream. +func ImmediateResponse(statusCode envoy_type.StatusCode, message string) *extprocv3.ProcessingResponse { + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ImmediateResponse{ + ImmediateResponse: &extprocv3.ImmediateResponse{ + Status: &envoy_type.HttpStatus{ + Code: statusCode, + }, + Body: []byte(message), + Headers: &extprocv3.HeaderMutation{ + SetHeaders: []*corev3.HeaderValueOption{ + { + // Using RawValues instead of Value: newer versions of Envoy + // drop Value and use RawValue + Header: &corev3.HeaderValue{ + Key: "content-type", + RawValue: []byte("text/plain"), + }, + }, + }, + }, + }, + }, + } +} diff --git a/cmd/atenet/internal/router/extproc/errors_test.go b/cmd/atenet/internal/router/extproc/errors_test.go new file mode 100644 index 000000000..ad8da7fe3 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/errors_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "errors" + "testing" + + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +func TestNewReqError(t *testing.T) { + t.Parallel() + + err := NewReqError(envoy_type.StatusCode_BadRequest, "actor %q is %s", "abc", "bad") + if err == nil { + t.Fatal("NewReqError returned nil") + } + var reqErr *ReqError + if !errors.As(err, &reqErr) { + t.Fatalf("errors.As(*ReqError) = false, want true; err type = %T", err) + } + if reqErr.StatusCode != int(envoy_type.StatusCode_BadRequest) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_BadRequest) + } + if got, want := err.Error(), `actor "abc" is bad`; got != want { + t.Errorf("Error() = %q, want %q", got, want) + } +} + +// TestImmediateResponseHeaderEncoding pins the RawValue encoding: Envoy drops +// plain Value in ext_proc header mutations, so a Value-encoded header reaches +// the client with an empty value (found live — content-type on every immediate +// response had been arriving empty). +func TestImmediateResponseHeaderEncoding(t *testing.T) { + t.Parallel() + + resp := ImmediateResponse(envoy_type.StatusCode_ServiceUnavailable, "body") + set := resp.GetImmediateResponse().GetHeaders().GetSetHeaders() + if len(set) != 1 { + t.Fatalf("SetHeaders count = %d, want 1", len(set)) + } + h := set[0].GetHeader() + if h.GetKey() != "content-type" || string(h.GetRawValue()) != "text/plain" { + t.Errorf("header = %q:%q (RawValue), want content-type:text/plain", h.GetKey(), h.GetRawValue()) + } + if h.GetValue() != "" { + t.Errorf("header uses Value (%q); must use RawValue only", h.GetValue()) + } +} diff --git a/cmd/atenet/internal/router/extproc/extproc.go b/cmd/atenet/internal/router/extproc/extproc.go new file mode 100644 index 000000000..f66fc6fd1 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/extproc.go @@ -0,0 +1,176 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extproc implements the Envoy external processing (ext_proc) gRPC +// server that the atenet router serves to its Envoy gateways. +// +// This package is the multiplexer and nothing else: it terminates the +// ext_proc stream, works out which direction — ingress or egress — a request +// arrived on, dispatches to the Handler registered for that direction, and +// records the latency and outcome. The routing and authentication policy for +// each direction lives in the sibling ingress and egress packages, which apply +// opposite trust models and are deliberately kept apart. +package extproc + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "time" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel/metric" + "google.golang.org/grpc" +) + +// Handlers maps each direction this server serves to the handler for it. A +// direction absent from the map is refused at the mux — see Server.Process. +type Handlers map[Direction]Handler + +// Server implements the Envoy external processing gRPC server, dispatching +// each request to the Handler for the direction it arrived on. +type Server struct { + port int + handlers Handlers + recorder *QueryRecorder + routeDuration metric.Float64Histogram +} + +// NewServer builds the ext_proc mux serving the given handlers. Passing a +// subset of the directions is how --mode restricts an instance to the traffic +// its deployment fronts. +func NewServer(port int, routeDuration metric.Float64Histogram, handlers Handlers) *Server { + return &Server{ + port: port, + handlers: handlers, + recorder: NewQueryRecorder(100), + routeDuration: routeDuration, + } +} + +// Queries returns the most recently processed requests, newest first, for the +// /statusz page. +func (s *Server) Queries() []RecordedQuery { + return s.recorder.Get() +} + +// Recorder exposes the query ring buffer so tests and the status page can seed +// or read it. +func (s *Server) Recorder() *QueryRecorder { return s.recorder } + +// NewGRPCServer builds the gRPC server with the ext_proc service registered. +// The caller owns its lifecycle: Run serves it, and the drain sequence in +// drain.go stops it — gracefully first so in-flight streams (parked requests +// above all) finish, forcefully past the drain timeout. +func (s *Server) NewGRPCServer() *grpc.Server { + grpcServer := grpc.NewServer( + grpc.StatsHandler(otelgrpc.NewServerHandler()), + ) + extprocv3.RegisterExternalProcessorServer(grpcServer, s) + return grpcServer +} + +func (s *Server) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + var resp *extprocv3.ProcessingResponse + + switch reqType := req.Request.(type) { + case *extprocv3.ProcessingRequest_RequestHeaders: + resp = s.processRequestHeaders(stream.Context(), req, reqType.RequestHeaders) + + default: + // No modification for other processing states, but log because this should + // not be called. + slog.Error("Unexpected request type", slog.String("reqType", fmt.Sprintf("%T", reqType))) + resp = &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, + } + } + + if err := stream.Send(resp); err != nil { + return err + } + } +} + +// processRequestHeaders runs one RequestHeaders callback through the handler +// for its direction, and records the latency and outcome either way. +func (s *Server) processRequestHeaders( + ctx context.Context, + req *extprocv3.ProcessingRequest, + reqHeaders *extprocv3.HttpHeaders, +) *extprocv3.ProcessingResponse { + start := time.Now() + md := NewRequestMetadata(reqHeaders.GetHeaders().GetHeaders()) + + // One atenet binary serves both directions, as two ext_proc handlers + // selected here. They are deployed separately today — atenet-router fronts + // the ingress Envoy, atenet-egress the egress one — because the two scale + // independently, and --mode restricts an instance to the direction its + // deployment fronts. Nothing stops a single instance from serving both. + // + // Which handler runs is decided by the Envoy filter chain that accepted the + // request, never by anything in the request itself (see directionOf). + dir := directionOf(req) + + var res Result + var err error + if handler, ok := s.handlers[dir]; ok { + res, err = handler.HandleRequestHeaders(ctx, md) + } else { + // The Envoy in front of this instance is sending traffic the instance + // was not started to serve. Refuse it rather than falling back to the + // other direction's handler: the two apply opposite trust models, so a + // fallback would run a request through the wrong one. + err = NewReqError(envoy_type.StatusCode_NotFound, + "this router does not serve %s traffic", dir) + } + + elapsed := time.Since(start) + s.recordRouteDuration(ctx, elapsed, res.TemplateNamespace, res.TemplateName, classifyOutcome(err), res.resume()) + + if err != nil { + slog.ErrorContext(ctx, "Error during ext_proc RequestHeaders processing", + slog.String("direction", string(dir)), + slog.String("err", err.Error())) + s.recorder.AddRouterRequest(start, elapsed, "Error", "-", md) + + var reqErr *ReqError + if errors.As(err, &reqErr) { + return ImmediateResponse(envoy_type.StatusCode(reqErr.StatusCode), reqErr.Error()) + } + return ImmediateResponse(envoy_type.StatusCode_InternalServerError, err.Error()) + } + + s.recorder.AddRouterRequest(start, elapsed, "Route ok", res.Target, md) + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{RequestHeaders: res.Response}, + } +} diff --git a/cmd/atenet/internal/router/extproc/extproc_test.go b/cmd/atenet/internal/router/extproc/extproc_test.go new file mode 100644 index 000000000..384b910cc --- /dev/null +++ b/cmd/atenet/internal/router/extproc/extproc_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + "strings" + "testing" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +// stubHandler records that it ran and returns an empty successful Result. +type stubHandler struct { + direction Direction + called bool +} + +func (h *stubHandler) Direction() Direction { return h.direction } + +func (h *stubHandler) HandleRequestHeaders(context.Context, *RequestMetadata) (Result, error) { + h.called = true + return Result{Response: &extprocv3.HeadersResponse{Response: &extprocv3.CommonResponse{}}}, nil +} + +// The mux must pick the handler by the Envoy-asserted filter chain, and refuse +// outright when this instance was not started to serve that direction (--mode). +// Falling back to the other handler would run the request through the opposite +// trust model. +func TestProcessRequestHeadersDispatchesByMode(t *testing.T) { + tests := []struct { + name string + registered []Direction + chain string + wantRan Direction + wantStatus envoy_type.StatusCode // 0 means "expect success" + }{ + { + name: "both directions served, egress chain", + registered: []Direction{DirectionIngress, DirectionEgress}, + chain: EgressFilterChainName, + wantRan: DirectionEgress, + }, + { + name: "both directions served, ingress chain", + registered: []Direction{DirectionIngress, DirectionEgress}, + chain: ingressHTTPListener, + wantRan: DirectionIngress, + }, + { + name: "egress-only instance refuses ingress traffic", + registered: []Direction{DirectionEgress}, + chain: ingressHTTPListener, + wantStatus: envoy_type.StatusCode_NotFound, + }, + { + name: "ingress-only instance refuses egress traffic", + registered: []Direction{DirectionIngress}, + chain: EgressFilterChainName, + wantStatus: envoy_type.StatusCode_NotFound, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + handlers := Handlers{} + stubs := map[Direction]*stubHandler{} + for _, d := range tc.registered { + stubs[d] = &stubHandler{direction: d} + handlers[d] = stubs[d] + } + + s := NewServer(50051, nil, handlers) + req := connectRequest("envoy.filters.http.ext_proc", tc.chain) + resp := s.processRequestHeaders(context.Background(), req, req.GetRequestHeaders()) + + if tc.wantStatus != 0 { + ir := resp.GetImmediateResponse() + if ir == nil { + t.Fatalf("expected an immediate response, got %v", resp) + } + if got := ir.GetStatus().GetCode(); got != tc.wantStatus { + t.Errorf("status = %v, want %v", got, tc.wantStatus) + } + if !strings.Contains(string(ir.GetBody()), "does not serve") { + t.Errorf("body = %q, want it to say the direction is not served", ir.GetBody()) + } + for d, h := range stubs { + if h.called { + t.Errorf("%s handler ran for a direction this instance does not serve", d) + } + } + return + } + + if resp.GetImmediateResponse() != nil { + t.Fatalf("unexpected immediate response: %v", resp.GetImmediateResponse()) + } + for d, h := range stubs { + if want := d == tc.wantRan; h.called != want { + t.Errorf("%s handler called = %v, want %v", d, h.called, want) + } + } + }) + } +} diff --git a/cmd/atenet/internal/router/extproc/handler.go b/cmd/atenet/internal/router/extproc/handler.go new file mode 100644 index 000000000..e0806631a --- /dev/null +++ b/cmd/atenet/internal/router/extproc/handler.go @@ -0,0 +1,74 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + + "github.com/agent-substrate/substrate/internal/ateattr" +) + +// Handler applies one direction's policy to a request. Implementations live in +// the ingress and egress packages; the mux never inspects what they do, only +// which direction they serve. +type Handler interface { + // Direction reports the traffic direction this handler serves. The mux keys + // its dispatch table by it. + Direction() Direction + + // HandleRequestHeaders decides what Envoy should do with a request whose + // headers have just arrived. + // + // A returned error denies the request: a *ReqError carries the status code + // and client-safe body to answer with, anything else becomes a 500. The + // Result is read even when an error is returned, so a handler that got far + // enough to learn the metric attributes (template identity, resume outcome) + // should still fill them in. + HandleRequestHeaders(ctx context.Context, md *RequestMetadata) (Result, error) +} + +// Result is what a handler tells the mux about a request it allowed. +type Result struct { + // Response is the header mutation Envoy applies before the request + // continues. Handlers that only authenticate return an empty CommonResponse. + Response *extprocv3.HeadersResponse + + // Target is the upstream address the request was routed to, shown on the + // /statusz page. Empty for handlers that do not pick an upstream. + Target string + + // TemplateNamespace and TemplateName identify the actor template the + // request resolved to. They are the low-cardinality attributes on the + // route-duration metric, and are empty when the direction has no template + // (or the request failed before resolving one). + TemplateNamespace string + TemplateName string + + // Resume is the actor-resume outcome, as one of the ateattr.RouterResume* + // values. Empty means "none" — the direction never resumes an actor, or the + // request never got that far. + Resume string +} + +// resume returns the resume label for the route-duration metric, defaulting an +// unset outcome to "none". +func (r Result) resume() string { + if r.Resume == "" { + return ateattr.RouterResumeNone + } + return r.Resume +} diff --git a/cmd/atenet/internal/router/extproc_in.go b/cmd/atenet/internal/router/extproc/metadata.go similarity index 52% rename from cmd/atenet/internal/router/extproc_in.go rename to cmd/atenet/internal/router/extproc/metadata.go index ce974983f..52f4fe7dd 100644 --- a/cmd/atenet/internal/router/extproc_in.go +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -12,26 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package extproc import ( - "net" "strings" - "github.com/agent-substrate/substrate/internal/resources" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" ) -const authorityHeader = ":authority" +// AuthorityHeader is the HTTP/2 pseudo-header carrying the request authority. +// Exported because the direction handlers rewrite it when they pick an +// upstream. +const AuthorityHeader = ":authority" -type requestMetadata struct { - headers map[string]string - path string - host string - method string +// RequestMetadata is the request the mux hands to a direction handler: the +// HTTP headers Envoy sent, flattened and lowercased, with the pseudo-headers +// every handler needs pulled out. +type RequestMetadata struct { + // Headers holds every header, keyed by lowercased name. + Headers map[string]string + Path string + Host string + Method string } -func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata { +func NewRequestMetadata(headers []*corev3.HeaderValue) *RequestMetadata { headersMap := make(map[string]string) var path string var host string @@ -48,7 +53,7 @@ func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata { if k == ":path" { path = val } - if k == authorityHeader || k == "host" { + if k == AuthorityHeader || k == "host" { host = val } if k == ":method" { @@ -56,26 +61,16 @@ func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata { } } - return &requestMetadata{ - headers: headersMap, - path: path, - host: host, - method: method, + return &RequestMetadata{ + Headers: headersMap, + Path: path, + Host: host, + Method: method, } } -// parseActorRef extracts the actor an incoming request is addressed to from its -// Host/:authority, which has the form -// "..actors.resources.substrate.ate.dev" (optionally with a -// port). The atespace is part of the name because an actor name is only unique -// within its atespace. -func parseActorRef(host string) (resources.ActorRef, error) { - if strings.Contains(host, ":") { - h, _, err := net.SplitHostPort(host) - if err != nil { - return resources.ActorRef{}, err - } - host = h - } - return resources.ParseActorDNSName(host) +// Header returns the value of a header by name, case-insensitively, or "" when +// it was not sent. +func (m *RequestMetadata) Header(name string) string { + return m.Headers[strings.ToLower(name)] } diff --git a/cmd/atenet/internal/router/extproc_in_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go similarity index 57% rename from cmd/atenet/internal/router/extproc_in_test.go rename to cmd/atenet/internal/router/extproc/metadata_test.go index bbb7a8f9e..5627afd0f 100644 --- a/cmd/atenet/internal/router/extproc_in_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package extproc import ( "reflect" "testing" - "github.com/agent-substrate/substrate/internal/resources" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" ) @@ -105,78 +104,16 @@ func TestExtractMetadata(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got := newRequestMetadata(tc.headers) + got := NewRequestMetadata(tc.headers) - if !reflect.DeepEqual(got.headers, tc.wantHeaders) { - t.Errorf("extractMetadata() headersMap = %v, want %v", got.headers, tc.wantHeaders) + if !reflect.DeepEqual(got.Headers, tc.wantHeaders) { + t.Errorf("NewRequestMetadata() headersMap = %v, want %v", got.Headers, tc.wantHeaders) } - if got.path != tc.wantPath { - t.Errorf("extractMetadata() path = %v, want %v", got.path, tc.wantPath) + if got.Path != tc.wantPath { + t.Errorf("NewRequestMetadata() path = %v, want %v", got.Path, tc.wantPath) } - if got.host != tc.wantHost { - t.Errorf("extractMetadata() host = %v, want %v", got.host, tc.wantHost) - } - }) - } -} - -func TestParseActorRef(t *testing.T) { - tests := []struct { - name string - host string - want resources.ActorRef - wantErr bool - }{ - { - name: "valid host without port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:8443", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot and port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.:8080", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "missing atespace label", - host: "my-actor.actors.resources.substrate.ate.dev", - wantErr: true, - }, - { - name: "invalid suffix", - host: "my-actor.team-a.example.com", - wantErr: true, - }, - { - name: "invalid host port format", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:invalid:port", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := parseActorRef(tc.host) - if (err != nil) != tc.wantErr { - t.Errorf("parseActorRef(%q) error = %v, wantErr %v", tc.host, err, tc.wantErr) - return - } - if got != tc.want { - t.Errorf("parseActorRef(%q) = %+v, want %+v", tc.host, got, tc.want) + if got.Host != tc.wantHost { + t.Errorf("NewRequestMetadata() host = %v, want %v", got.Host, tc.wantHost) } }) } diff --git a/cmd/atenet/internal/router/extproc/metrics.go b/cmd/atenet/internal/router/extproc/metrics.go new file mode 100644 index 000000000..8ba0819e7 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/metrics.go @@ -0,0 +1,111 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + "errors" + "fmt" + "time" + + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/ateattr" +) + +// ServiceName is the OpenTelemetry service name and instrumentation scope +// shared by every part of the atenet router process — the ext_proc mux here, +// the direction handlers, and the router's own servers. It lives in this +// package because it is the one package all of them already depend on. +const ServiceName = "atenet-router" + +// atenet.router.route.duration measures the latency from when the ext_proc handler receives a request +// (Envoy -> EPP) until the target worker endpoint is resolved +const routeDurationMetricName = "atenet.router.route.duration" + +// NewRouteDurationHistogram creates the atenet.router.route.duration histogram from +// the global MeterProvider. +func NewRouteDurationHistogram() (metric.Float64Histogram, error) { + h, err := otel.Meter(ServiceName).Float64Histogram( + routeDurationMetricName, + metric.WithUnit("s"), + metric.WithDescription( + "latency between Substrate router receiving a request and resolving "+ + "the target worker endpoint, excluding actor execution and response", + ), + metric.WithExplicitBucketBoundaries( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, + 0.075, 0.1, 0.15, 0.2, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30, + ), + ) + if err != nil { + return nil, fmt.Errorf("create %s histogram: %w", routeDurationMetricName, err) + } + return h, nil +} + +func (s *Server) recordRouteDuration(ctx context.Context, d time.Duration, tmplNs, tmplName, outcome, resume string) { + if s.routeDuration == nil { + return + } + s.routeDuration.Record(ctx, d.Seconds(), metric.WithAttributes( + ateattr.TemplateNamespaceKey.String(tmplNs), + ateattr.TemplateNameKey.String(tmplName), + ateattr.RouterOutcomeKey.String(outcome), + ateattr.RouterResumeKey.String(resume), + )) +} + +func classifyOutcome(err error) string { + if err == nil { + return "ok" + } + if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { + return "cancelled" + } + if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded { + return "timeout" + } + switch status.Code(err) { + case codes.FailedPrecondition: + return "no_capacity" + case codes.Aborted: + return "lock_conflict" + case codes.NotFound: + return "not_found" + case codes.Unavailable: + return "unavailable" + case codes.ResourceExhausted: + return "rate_limited" + } + var re *ReqError + if errors.As(err, &re) { + switch envoy_type.StatusCode(re.StatusCode) { + case envoy_type.StatusCode_NotFound: + return "not_found" + case envoy_type.StatusCode_ServiceUnavailable: + return "no_capacity" + case envoy_type.StatusCode_GatewayTimeout: + return "timeout" + case envoy_type.StatusCode_TooManyRequests: + return "rate_limited" + } + } + return "resume_error" +} diff --git a/cmd/atenet/internal/router/extproc/metrics_test.go b/cmd/atenet/internal/router/extproc/metrics_test.go new file mode 100644 index 000000000..c35ea705c --- /dev/null +++ b/cmd/atenet/internal/router/extproc/metrics_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "context" + "errors" + "testing" + "time" + + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/internal/ateattr" +) + +func TestClassifyOutcome(t *testing.T) { + tests := []struct { + name string + err error + expected string + }{ + { + name: "nil error maps to ok", + err: nil, + expected: "ok", + }, + { + name: "context Canceled maps to cancelled", + err: context.Canceled, + expected: "cancelled", + }, + { + name: "context DeadlineExceeded maps to timeout", + err: context.DeadlineExceeded, + expected: "timeout", + }, + { + name: "FailedPrecondition gRPC code maps to no_capacity", + err: status.Error(codes.FailedPrecondition, "capacity full"), + expected: "no_capacity", + }, + { + name: "Aborted gRPC code maps to lock_conflict", + err: status.Error(codes.Aborted, "lock conflict"), + expected: "lock_conflict", + }, + { + name: "NotFound gRPC code maps to not_found", + err: status.Error(codes.NotFound, "missing"), + expected: "not_found", + }, + { + name: "Unavailable gRPC code maps to unavailable", + err: status.Error(codes.Unavailable, "control-plane down"), + expected: "unavailable", + }, + { + name: "ResourceExhausted gRPC code maps to rate_limited", + err: status.Error(codes.ResourceExhausted, "rate limit exceeded"), + expected: "rate_limited", + }, + { + name: "StatusCode_NotFound ReqError maps to not_found", + err: NewReqError(envoy_type.StatusCode_NotFound, "missing"), + expected: "not_found", + }, + { + name: "StatusCode_ServiceUnavailable ReqError maps to no_capacity", + err: NewReqError(envoy_type.StatusCode_ServiceUnavailable, "no free workers"), + expected: "no_capacity", + }, + { + name: "StatusCode_TooManyRequests ReqError maps to rate_limited", + err: NewReqError(envoy_type.StatusCode_TooManyRequests, "rate limited"), + expected: "rate_limited", + }, + { + name: "Unknown error maps to resume_error", + err: errors.New("internal storage glitch"), + expected: "resume_error", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := classifyOutcome(tc.err); got != tc.expected { + t.Errorf("classifyOutcome(%v) = %q, want %q", tc.err, got, tc.expected) + } + }) + } +} + +func TestRecordRouteDuration_Attributes(t *testing.T) { + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + h, err := mp.Meter("atenet-router").Float64Histogram(routeDurationMetricName) + if err != nil { + t.Fatalf("failed to create histogram: %v", err) + } + + s := NewServer(50051, h, nil) + s.recordRouteDuration(context.Background(), 10*time.Millisecond, "team-a-ns", "tmpl-a", classifyOutcome(nil), ateattr.RouterResumeTriggered) + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect failed: %v", err) + } + + dp := rm.ScopeMetrics[0].Metrics[0].Data.(metricdata.Histogram[float64]).DataPoints[0] + wantAttrs := map[string]string{ + "ate.template.namespace": "team-a-ns", + "ate.template.name": "tmpl-a", + "ate.router.outcome": "ok", + "ate.router.resume": "triggered", + } + + for k, want := range wantAttrs { + val, exists := dp.Attributes.Value(attribute.Key(k)) + if !exists { + t.Errorf("missing metric attribute %q", k) + } else if val.AsString() != want { + t.Errorf("attribute %q = %q, want %q", k, val.AsString(), want) + } + } +} diff --git a/cmd/atenet/internal/router/extproc/record.go b/cmd/atenet/internal/router/extproc/record.go new file mode 100644 index 000000000..51c6b8774 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/record.go @@ -0,0 +1,117 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "strings" + "sync" + "time" +) + +type RecordedQuery struct { + Timestamp time.Time `json:"timestamp"` + Client string `json:"client"` + Host string `json:"host"` + Path string `json:"path"` + Method string `json:"method"` + Action string `json:"action"` + Target string `json:"target"` + Duration time.Duration `json:"duration"` +} + +// QueryRecorder is the fixed-size ring of recently processed requests behind +// the router's /statusz page. +type QueryRecorder struct { + mu sync.RWMutex + queries []RecordedQuery + size int + index int +} + +func NewQueryRecorder(size int) *QueryRecorder { + return &QueryRecorder{ + queries: make([]RecordedQuery, 0, size), + size: size, + } +} + +func (qr *QueryRecorder) Add(q RecordedQuery) { + if qr == nil { + return + } + + qr.mu.Lock() + defer qr.mu.Unlock() + + if len(qr.queries) < qr.size { + qr.queries = append(qr.queries, q) + } else { + qr.queries[qr.index] = q + qr.index = (qr.index + 1) % qr.size + } +} + +func (qr *QueryRecorder) Get() []RecordedQuery { + if qr == nil { + return nil + } + + qr.mu.RLock() + defer qr.mu.RUnlock() + + n := len(qr.queries) + if n == 0 { + return nil + } + + res := make([]RecordedQuery, n) + if n < qr.size { + for i := 0; i < n; i++ { + res[i] = qr.queries[n-1-i] + } + } else { + for i := 0; i < n; i++ { + pos := (qr.index - 1 - i + n) % n + res[i] = qr.queries[pos] + } + } + + return res +} + +// redactPath drops the query string, which may carry credentials (CWE-598). +func redactPath(path string) string { + p, _, _ := strings.Cut(path, "?") + return p +} + +func (qr *QueryRecorder) AddRouterRequest( + start time.Time, + duration time.Duration, + action, + target string, + m *RequestMetadata, +) { + qr.Add(RecordedQuery{ + Timestamp: start, + Client: m.Headers[AuthorityHeader], + Host: m.Host, + Path: redactPath(m.Path), + Method: m.Headers[":method"], + Action: action, + Target: target, + Duration: duration, + }) +} diff --git a/cmd/atenet/internal/router/extproc_out.go b/cmd/atenet/internal/router/extproc_out.go deleted file mode 100644 index a6759daf6..000000000 --- a/cmd/atenet/internal/router/extproc_out.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package router - -import ( - "github.com/agent-substrate/substrate/internal/atunnel" - corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" - envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" -) - -// reqError carries an HTTP-mappable status code and a client-safe message. -// The underlying cause (if any) is preserved via Unwrap so logs can inspect -// the full chain without leaking server-side detail into the response body. -type reqError struct { - msg string - cause error - statusCode int -} - -func (e *reqError) Error() string { return e.msg } -func (e *reqError) Unwrap() error { return e.cause } - -// addOriginalDstMutation sets the header the ORIGINAL_DST cluster reads to pick -// the upstream address (the worker atunnel IP:443). Unlike an :authority -// rewrite it leaves the request Host intact, so atunnel still sees the actor -// DNS name and can authorize the active actor. -// -// Nothing strips this header from the incoming request, so overwrite rather -// than append: a client-supplied value must never influence the address Envoy -// dials. ext_proc mutations already default to replace, but the default is -// split across the deprecated append field and append_action — pin it. -func addOriginalDstMutation(dst string, mut *extproc.HeaderMutation) { - mut.SetHeaders = append(mut.SetHeaders, - &corev3.HeaderValueOption{ - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - Header: &corev3.HeaderValue{ - Key: OriginalDstHeader, - RawValue: []byte(dst), - }, - }, - ) -} - -// addRoutingMutations overwrites all routing metadata derived from the -// control-plane result. Envoy dials OriginalDstHeader while preserving -// :authority. Agentgateway v1.4.1's static dynamic backend instead dials the -// request :authority, so that mode rewrites it to the worker atunnel address. -// OriginalHostHeader lets atunnel restore and authorize the actor authority. -func addRoutingMutations(dst, actorHost string, routeViaAuthority bool, mut *extproc.HeaderMutation) { - addOriginalDstMutation(dst, mut) - mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - Header: &corev3.HeaderValue{ - Key: atunnel.OriginalHostHeader, - RawValue: []byte(actorHost), - }, - }) - if routeViaAuthority { - mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - Header: &corev3.HeaderValue{ - Key: authorityHeader, - RawValue: []byte(dst), - }, - }) - } -} - -func immediateResponse(statusCode envoy_type.StatusCode, message string) *extproc.ProcessingResponse { - return &extproc.ProcessingResponse{ - Response: &extproc.ProcessingResponse_ImmediateResponse{ - ImmediateResponse: &extproc.ImmediateResponse{ - Status: &envoy_type.HttpStatus{ - Code: statusCode, - }, - Body: []byte(message), - Headers: &extproc.HeaderMutation{ - SetHeaders: []*corev3.HeaderValueOption{ - { - // Using RawValues instead of Value: newer versions of Envoy - // drop Value and use RawValue - Header: &corev3.HeaderValue{ - Key: "content-type", - RawValue: []byte("text/plain"), - }, - }, - }, - }, - }, - }, - } -} diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index 49ee002a0..99602fe64 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -157,6 +157,14 @@ func updateComponentHealth(health *ComponentHealth, healthy bool, msg string, ch } func (rh *routerHealth) checkDataplane(ctx context.Context) (bool, string) { + // The dataplane this polls is the *ingress* proxy sharing the router's pod. + // The egress Envoy is a separate, statically configured proxy on its own + // admin port; an egress-only router has none beside it, and probing this + // address would report a permanently unhealthy dependency. + if !rh.cfg.Mode.ServesIngress() { + return true, "Skipped (egress mode)" + } + timeoutCtx, cancel := context.WithTimeout(ctx, dependencyHealthCheckTimeout) defer cancel() diff --git a/cmd/atenet/internal/router/health_test.go b/cmd/atenet/internal/router/health_test.go index e1b4c0393..dd594052b 100644 --- a/cmd/atenet/internal/router/health_test.go +++ b/cmd/atenet/internal/router/health_test.go @@ -16,6 +16,7 @@ package router import ( "context" + "errors" "io" "net/http" "net/http/httptest" @@ -112,6 +113,25 @@ func TestCheckDataplane(t *testing.T) { } } +// An egress-only router has no ingress dataplane beside it, so probing the +// ingress proxy's admin port would report a dependency that is permanently +// down. +func TestCheckDataplaneSkippedInEgressMode(t *testing.T) { + rh := newRouterHealth(time.Second, nil, nil, routerConfig{Mode: ModeEgress}) + rh.dataplaneClient = &http.Client{Transport: healthRoundTripFunc(func(*http.Request) (*http.Response, error) { + t.Error("checkDataplane dialed the ingress dataplane admin port in egress mode") + return nil, errors.New("unexpected request") + })} + + healthy, msg := rh.checkDataplane(context.Background()) + if !healthy { + t.Errorf("checkDataplane healthy = false, want true (skipped)") + } + if !strings.Contains(msg, "Skipped") { + t.Errorf("checkDataplane message = %q, want it to report the check was skipped", msg) + } +} + func TestCheckK8sTimesOut(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { <-req.Context().Done() diff --git a/cmd/atenet/internal/router/ingress/actorref_test.go b/cmd/atenet/internal/router/ingress/actorref_test.go new file mode 100644 index 000000000..a87639f24 --- /dev/null +++ b/cmd/atenet/internal/router/ingress/actorref_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import ( + "testing" + + "github.com/agent-substrate/substrate/internal/resources" +) + +func TestParseActorRef(t *testing.T) { + tests := []struct { + name string + host string + want resources.ActorRef + wantErr bool + }{ + { + name: "valid host without port", + host: "my-actor.team-a.actors.resources.substrate.ate.dev", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "valid host with port", + host: "my-actor.team-a.actors.resources.substrate.ate.dev:8443", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "valid host with trailing dot", + host: "my-actor.team-a.actors.resources.substrate.ate.dev.", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "valid host with trailing dot and port", + host: "my-actor.team-a.actors.resources.substrate.ate.dev.:8080", + want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, + wantErr: false, + }, + { + name: "missing atespace label", + host: "my-actor.actors.resources.substrate.ate.dev", + wantErr: true, + }, + { + name: "invalid suffix", + host: "my-actor.team-a.example.com", + wantErr: true, + }, + { + name: "invalid host port format", + host: "my-actor.team-a.actors.resources.substrate.ate.dev:invalid:port", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseActorRef(tc.host) + if (err != nil) != tc.wantErr { + t.Errorf("parseActorRef(%q) error = %v, wantErr %v", tc.host, err, tc.wantErr) + return + } + if got != tc.want { + t.Errorf("parseActorRef(%q) = %+v, want %+v", tc.host, got, tc.want) + } + }) + } +} diff --git a/cmd/atenet/internal/router/errors.go b/cmd/atenet/internal/router/ingress/errors.go similarity index 60% rename from cmd/atenet/internal/router/errors.go rename to cmd/atenet/internal/router/ingress/errors.go index 251148747..5c08f57a2 100644 --- a/cmd/atenet/internal/router/errors.go +++ b/cmd/atenet/internal/router/ingress/errors.go @@ -12,41 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" "errors" "fmt" - "github.com/agent-substrate/substrate/internal/resources" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" -) -// newReqError builds a reqError whose body is the formatted message and no -// wrapped cause. Set the cause field directly when one is available. -func newReqError(code envoy_type.StatusCode, format string, args ...any) error { - return &reqError{ - msg: fmt.Sprintf(format, args...), - statusCode: int(code), - } -} + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/resources" +) -// actorNotFoundErr returns a 404 reqError identifying the missing actor. +// actorNotFoundErr returns a 404 denial identifying the missing actor. func actorNotFoundErr(actorRef resources.ActorRef) error { - return newReqError(envoy_type.StatusCode_NotFound, "actor %s not found", actorRef) + return extproc.NewReqError(envoy_type.StatusCode_NotFound, "actor %s not found", actorRef) } -// invalidHostErr returns a 404 reqError explaining why the request host was +// invalidHostErr returns a 404 denial explaining why the request host was // rejected. The cause is preserved for log inspection via Unwrap. func invalidHostErr(host string, cause error) error { - return &reqError{ - msg: fmt.Sprintf("invalid host %q: %v", host, cause), - cause: cause, - statusCode: int(envoy_type.StatusCode_NotFound), - } + return extproc.WrapReqError(envoy_type.StatusCode_NotFound, cause, "invalid host %q: %v", host, cause) } // statusDescription returns the gRPC status description of err, unwrapping @@ -62,15 +51,15 @@ func statusDescription(err error) string { return status.Convert(err).Message() } -// parkingFullErr returns a 503 reqError signaling that the router's parking lot +// parkingFullErr returns a 503 denial signaling that the router's parking lot // is at capacity, so the request was shed without waiting. Clients should retry. func parkingFullErr(actorID string) error { - return newReqError(envoy_type.StatusCode_ServiceUnavailable, + return extproc.NewReqError(envoy_type.StatusCode_ServiceUnavailable, "actor %q unavailable: router at capacity", actorID) } // mapResumeError translates an ActorResumer error into a client-facing -// reqError. It maps gRPC status codes to appropriate HTTP status codes and +// denial. It maps gRPC status codes to appropriate HTTP status codes and // short, human-readable bodies. The original error is preserved via Unwrap // so callers can still inspect it via errors.Is / errors.As when logging. // @@ -81,7 +70,7 @@ func mapResumeError(actorRef resources.ActorRef, err error) error { return nil } - re := &reqError{cause: err} + re := &extproc.ReqError{Cause: err} // Bare context sentinels reach here when the request's own context ends // (client disconnect or stream deadline) — status.Code would classify them @@ -90,50 +79,50 @@ func mapResumeError(actorRef resources.ActorRef, err error) error { // already dead, so the code is observability-only; Envoy's StatusCode enum // has no 499 ("client closed request"), so 408 is the nearest defined code. if errors.Is(err, context.Canceled) { - re.statusCode = int(envoy_type.StatusCode_RequestTimeout) - re.msg = fmt.Sprintf("request for actor %s canceled by client", actorRef) + re.StatusCode = int(envoy_type.StatusCode_RequestTimeout) + re.Msg = fmt.Sprintf("request for actor %s canceled by client", actorRef) return re } if errors.Is(err, context.DeadlineExceeded) { - re.statusCode = int(envoy_type.StatusCode_GatewayTimeout) - re.msg = fmt.Sprintf("actor %s request timed out", actorRef) + re.StatusCode = int(envoy_type.StatusCode_GatewayTimeout) + re.Msg = fmt.Sprintf("actor %s request timed out", actorRef) return re } switch status.Code(err) { case codes.NotFound: - re.statusCode = int(envoy_type.StatusCode_NotFound) - re.msg = fmt.Sprintf("actor %s not found", actorRef) + re.StatusCode = int(envoy_type.StatusCode_NotFound) + re.Msg = fmt.Sprintf("actor %s not found", actorRef) case codes.FailedPrecondition: // Preserve the gRPC description for FailedPrecondition and Aborted: // they carry actionable client-facing context (e.g. "no free workers // available", "another operation is in progress for this actor") and // are not security-sensitive. - re.statusCode = int(envoy_type.StatusCode_ServiceUnavailable) - re.msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) + re.StatusCode = int(envoy_type.StatusCode_ServiceUnavailable) + re.Msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) case codes.Aborted: // A concurrency conflict that outlived its retries (e.g. a park budget // spent entirely on Aborted). Retryable by the client, hence 503. - re.statusCode = int(envoy_type.StatusCode_ServiceUnavailable) - re.msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) + re.StatusCode = int(envoy_type.StatusCode_ServiceUnavailable) + re.Msg = fmt.Sprintf("actor %s unavailable: %s", actorRef, statusDescription(err)) case codes.Unavailable: - re.statusCode = int(envoy_type.StatusCode_ServiceUnavailable) - re.msg = fmt.Sprintf("actor %s unavailable", actorRef) + re.StatusCode = int(envoy_type.StatusCode_ServiceUnavailable) + re.Msg = fmt.Sprintf("actor %s unavailable", actorRef) case codes.DeadlineExceeded: - re.statusCode = int(envoy_type.StatusCode_GatewayTimeout) - re.msg = fmt.Sprintf("actor %s request timed out", actorRef) + re.StatusCode = int(envoy_type.StatusCode_GatewayTimeout) + re.Msg = fmt.Sprintf("actor %s request timed out", actorRef) case codes.PermissionDenied: - re.statusCode = int(envoy_type.StatusCode_Forbidden) - re.msg = fmt.Sprintf("actor %s access denied", actorRef) + re.StatusCode = int(envoy_type.StatusCode_Forbidden) + re.Msg = fmt.Sprintf("actor %s access denied", actorRef) case codes.Unauthenticated: - re.statusCode = int(envoy_type.StatusCode_Unauthorized) - re.msg = fmt.Sprintf("actor %s authentication required", actorRef) + re.StatusCode = int(envoy_type.StatusCode_Unauthorized) + re.Msg = fmt.Sprintf("actor %s authentication required", actorRef) case codes.ResourceExhausted: - re.statusCode = int(envoy_type.StatusCode_TooManyRequests) - re.msg = fmt.Sprintf("actor %s rate limited", actorRef) + re.StatusCode = int(envoy_type.StatusCode_TooManyRequests) + re.Msg = fmt.Sprintf("actor %s rate limited", actorRef) default: - re.statusCode = int(envoy_type.StatusCode_InternalServerError) - re.msg = fmt.Sprintf("error resuming actor %s", actorRef) + re.StatusCode = int(envoy_type.StatusCode_InternalServerError) + re.Msg = fmt.Sprintf("error resuming actor %s", actorRef) } return re } diff --git a/cmd/atenet/internal/router/errors_test.go b/cmd/atenet/internal/router/ingress/errors_test.go similarity index 72% rename from cmd/atenet/internal/router/errors_test.go rename to cmd/atenet/internal/router/ingress/errors_test.go index 3de804c7a..0e82f4af9 100644 --- a/cmd/atenet/internal/router/errors_test.go +++ b/cmd/atenet/internal/router/ingress/errors_test.go @@ -12,48 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" "errors" "testing" - "github.com/agent-substrate/substrate/internal/resources" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" -) -func TestNewReqError(t *testing.T) { - t.Parallel() - - err := newReqError(envoy_type.StatusCode_BadRequest, "actor %q is %s", "abc", "bad") - if err == nil { - t.Fatal("newReqError returned nil") - } - var reqErr *reqError - if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) - } - if reqErr.statusCode != int(envoy_type.StatusCode_BadRequest) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, envoy_type.StatusCode_BadRequest) - } - if got, want := err.Error(), `actor "abc" is bad`; got != want { - t.Errorf("Error() = %q, want %q", got, want) - } -} + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/resources" +) func TestActorNotFoundErr(t *testing.T) { t.Parallel() err := actorNotFoundErr(resources.ActorRef{Atespace: "team-a", Name: "ctr6"}) - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) } - if reqErr.statusCode != int(envoy_type.StatusCode_NotFound) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, envoy_type.StatusCode_NotFound) + if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) } if got, want := err.Error(), `actor team-a/ctr6 not found`; got != want { t.Errorf("Error() = %q, want %q", got, want) @@ -66,12 +49,12 @@ func TestInvalidHostErr(t *testing.T) { cause := errors.New("missing suffix") err := invalidHostErr("foo.example.com", cause) - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) } - if reqErr.statusCode != int(envoy_type.StatusCode_NotFound) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, envoy_type.StatusCode_NotFound) + if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) } if got, want := err.Error(), `invalid host "foo.example.com": missing suffix`; got != want { t.Errorf("Error() = %q, want %q", got, want) @@ -186,12 +169,12 @@ func TestMapResumeError(t *testing.T) { if got == nil { t.Fatal("mapResumeError returned nil") } - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(got, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", got) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", got) } - if reqErr.statusCode != int(tc.wantCode) { - t.Errorf("statusCode = %d, want %d", reqErr.statusCode, tc.wantCode) + if reqErr.StatusCode != int(tc.wantCode) { + t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, tc.wantCode) } if got.Error() != tc.wantBody { t.Errorf("Error() = %q, want %q", got.Error(), tc.wantBody) @@ -213,35 +196,14 @@ func TestMapResumeError_NilError(t *testing.T) { } } -// Ensures mapResumeError result satisfies the reqError contract so the -// existing handleRequestHeaders branch (errors.As(err, &reqErr)) keeps working. +// Ensures mapResumeError result satisfies the extproc.ReqError contract so the +// mux's errors.As(err, &reqErr) branch keeps mapping it to the right status. func TestMapResumeError_IsReqError(t *testing.T) { t.Parallel() err := mapResumeError(resources.ActorRef{Atespace: "team-a", Name: "x"}, status.Error(codes.NotFound, "x")) - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*reqError) = false, want true; err type = %T", err) - } -} - -// TestImmediateResponseHeaderEncoding pins the RawValue encoding: Envoy drops -// plain Value in ext_proc header mutations, so a Value-encoded header reaches -// the client with an empty value (found live — content-type on every immediate -// response had been arriving empty). -func TestImmediateResponseHeaderEncoding(t *testing.T) { - t.Parallel() - - resp := immediateResponse(envoy_type.StatusCode_ServiceUnavailable, "body") - set := resp.GetImmediateResponse().GetHeaders().GetSetHeaders() - if len(set) != 1 { - t.Fatalf("SetHeaders count = %d, want 1", len(set)) - } - h := set[0].GetHeader() - if h.GetKey() != "content-type" || string(h.GetRawValue()) != "text/plain" { - t.Errorf("header = %q:%q (RawValue), want content-type:text/plain", h.GetKey(), h.GetRawValue()) - } - if h.GetValue() != "" { - t.Errorf("header uses Value (%q); must use RawValue only", h.GetValue()) + t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) } } diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go new file mode 100644 index 000000000..30e3ca62e --- /dev/null +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -0,0 +1,211 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ingress implements the ext_proc handler for traffic entering the +// mesh: it resolves the actor a request is addressed to, resumes it through +// the control plane (parking the request while the worker pool is saturated), +// and points Envoy at the worker that ends up hosting it. +// +// Everything reaching this handler is unauthenticated client input. The +// opposite trust model — an actor identity carried by a CA-signed client +// certificate — belongs to the sibling egress package, and the two are kept +// apart deliberately. +package ingress + +import ( + "context" + "log/slog" + "net" + "strings" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// OriginalDstHeader carries the resolved worker atunnel address (IP:443) to +// the ORIGINAL_DST cluster. It is this handler's contract with the ingress +// Envoy configuration the xDS server generates. +const OriginalDstHeader = "x-ate-original-dst" + +// Handler routes ingress requests to the worker hosting their actor. +type Handler struct { + resumer *ActorResumer + parking *parkingLot + // routeViaAuthority rewrites :authority to the worker atunnel address for + // data planes that dial it rather than OriginalDstHeader. See + // addRoutingMutations. + routeViaAuthority bool +} + +func New(apiClient ateapipb.ControlClient, parkCfg ParkedRequestConfig, parkMetrics *ParkingMetrics, routeViaAuthority bool) *Handler { + return &Handler{ + resumer: NewActorResumer(apiClient, withParking(parkCfg)), + parking: newParkingLot(parkCfg, parkMetrics), + routeViaAuthority: routeViaAuthority, + } +} + +func (h *Handler) Direction() extproc.Direction { return extproc.DirectionIngress } + +// ParkingStatus returns a snapshot of the parking lot for the /statusz page. +func (h *Handler) ParkingStatus() ParkingStatus { return h.parking.status() } + +func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestMetadata) (extproc.Result, error) { + slog.InfoContext(ctx, "Request", slog.String("host", md.Host)) + + // Envoy doesn't propagate trace context into the ext_proc gRPC + // stream's metadata — the per-request traceparent arrives in the + // HTTP headers carried inside the ProcessingRequest payload. Extract + // from there so our span links to the Envoy ingress span. + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(md.Headers)) + ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") + defer span.End() + + actorRef, err := parseActorRef(md.Host) + if err != nil { + // Host is invalid, respond with 404. + return extproc.Result{}, invalidHostErr(md.Host, err) + } + + // Admit the request to the parking lot before resuming. While resume is + // in-flight the request occupies a slot; if the actor's worker pool is + // momentarily saturated the resumer parks (retries) here rather than failing + // fast. A full lot sheds the request immediately so the router applies + // backpressure instead of queueing without bound. + release, ok := h.parking.enter(ctx) + if !ok { + return extproc.Result{}, parkingFullErr(actorRef.String()) + } + + slog.InfoContext(ctx, "ResumeActor", slog.Any("actor", actorRef)) + actor, resumeOutcome, err := h.resumer.ResumeActor(ctx, actorRef) + release(parkOutcomeFor(err)) + if err != nil { + return extproc.Result{Resume: string(resumeOutcome)}, mapResumeError(actorRef, err) + } + + // Actor template identity, used as low-cardinality route-latency metric + // attributes. + res := extproc.Result{ + TemplateNamespace: actor.GetActorTemplateNamespace(), + TemplateName: actor.GetActorTemplateName(), + Resume: string(resumeOutcome), + } + + workerIP := actor.GetWorkerAssignment().GetWorkerPodIp() + slog.InfoContext(ctx, "ResumeActor result", + slog.Any("actor", actorRef), + slog.String("status", actor.GetStatus().String()), + slog.String("workerIP", workerIP)) + + if ip := net.ParseIP(workerIP); ip == nil { + return res, extproc.NewReqError(envoy_type.StatusCode_InternalServerError, + "actor %s routing failed", actorRef) + } + + // The actor is reached through the in-worker atunnel ingress server, which + // listens on :443 (mTLS) and forwards to the actor's :80. The worker no + // longer DNATs pod-IP:80 to the actor, so the router dials :443 and the + // ORIGINAL_DST cluster's upstream TLS context presents the router's + // podidentity client cert (see buildOriginalDstCluster and + // buildUpstreamTransportSocket). + // TODO(bowei) -- handle more than port 80 on the actor. + targetAddr := net.JoinHostPort(workerIP, "443") + + slog.InfoContext(ctx, "Route ok", slog.Any("actor", actorRef), slog.String("targetAddr", targetAddr)) + + // Route by telling the ORIGINAL_DST cluster which worker atunnel address to + // dial, without touching :authority — atunnel authorizes the actor by the + // original Host (actor DNS name). + mutation := &extprocv3.HeaderMutation{} + addRoutingMutations(targetAddr, md.Host, h.routeViaAuthority, mutation) + + res.Target = targetAddr + res.Response = &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{ + HeaderMutation: mutation, + }, + } + return res, nil +} + +// parseActorRef extracts the actor an incoming request is addressed to from its +// Host/:authority, which has the form +// "..actors.resources.substrate.ate.dev" (optionally with a +// port). The atespace is part of the name because an actor name is only unique +// within its atespace. +func parseActorRef(host string) (resources.ActorRef, error) { + if strings.Contains(host, ":") { + h, _, err := net.SplitHostPort(host) + if err != nil { + return resources.ActorRef{}, err + } + host = h + } + return resources.ParseActorDNSName(host) +} + +// addOriginalDstMutation sets the header the ORIGINAL_DST cluster reads to pick +// the upstream address (the worker atunnel IP:443). Unlike an :authority +// rewrite it leaves the request Host intact, so atunnel still sees the actor +// DNS name and can authorize the active actor. +// +// Nothing strips this header from the incoming request, so overwrite rather +// than append: a client-supplied value must never influence the address Envoy +// dials. ext_proc mutations already default to replace, but the default is +// split across the deprecated append field and append_action — pin it. +func addOriginalDstMutation(dst string, mut *extprocv3.HeaderMutation) { + mut.SetHeaders = append(mut.SetHeaders, + &corev3.HeaderValueOption{ + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + Header: &corev3.HeaderValue{ + Key: OriginalDstHeader, + RawValue: []byte(dst), + }, + }, + ) +} + +// addRoutingMutations overwrites all routing metadata derived from the +// control-plane result. Envoy dials OriginalDstHeader while preserving +// :authority. Agentgateway v1.4.1's static dynamic backend instead dials the +// request :authority, so that mode rewrites it to the worker atunnel address. +// OriginalHostHeader lets atunnel restore and authorize the actor authority. +func addRoutingMutations(dst, actorHost string, routeViaAuthority bool, mut *extprocv3.HeaderMutation) { + addOriginalDstMutation(dst, mut) + mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + Header: &corev3.HeaderValue{ + Key: atunnel.OriginalHostHeader, + RawValue: []byte(actorHost), + }, + }) + if routeViaAuthority { + mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{ + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + Header: &corev3.HeaderValue{ + Key: extproc.AuthorityHeader, + RawValue: []byte(dst), + }, + }) + } +} diff --git a/cmd/atenet/internal/router/extproc_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go similarity index 61% rename from cmd/atenet/internal/router/extproc_test.go rename to cmd/atenet/internal/router/ingress/ingress_test.go index cf3901521..1620a1060 100644 --- a/cmd/atenet/internal/router/extproc_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "bytes" @@ -24,17 +24,16 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atunnel" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "go.opentelemetry.io/otel/attribute" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/metric/metricdata" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) type mockClient struct { @@ -46,6 +45,12 @@ func (m *mockClient) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRe return m.resumeFn(ctx, in, opts...) } +// requestMetadata builds the metadata the ext_proc mux would hand the handler +// for a request with these headers. +func requestMetadata(headers ...*corev3.HeaderValue) *extproc.RequestMetadata { + return extproc.NewRequestMetadata(headers) +} + func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" const secret = "do-not-log-me" @@ -55,25 +60,21 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, nil))) t.Cleanup(func() { slog.SetDefault(prev) }) - s := NewExtProcServer(50051, &mockClient{ + h := New(&mockClient{ resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}}, nil }, - }, nil, ParkedRequestConfig{}, nil, false, nil) + }, ParkedRequestConfig{}, nil, false) - reqHeaders := &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":path", Value: "/api/v1/reset?token=" + secret}, - {Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, - {Key: ":method", Value: "POST"}, - {Key: "authorization", Value: "Bearer " + secret}, - {Key: "cookie", Value: "session=" + secret}, - }, - }, - } + md := requestMetadata( + &corev3.HeaderValue{Key: ":path", Value: "/api/v1/reset?token=" + secret}, + &corev3.HeaderValue{Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, + &corev3.HeaderValue{Key: ":method", Value: "POST"}, + &corev3.HeaderValue{Key: "authorization", Value: "Bearer " + secret}, + &corev3.HeaderValue{Key: "cookie", Value: "session=" + secret}, + ) - _, metadata, target, _, _, _, err := s.handleRequestHeaders(context.Background(), reqHeaders) + res, err := h.HandleRequestHeaders(context.Background(), md) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -86,15 +87,18 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { t.Errorf("router log missing actor/host routing context: %s", out) } - s.recorder.AddRouterRequest(time.Now(), time.Millisecond, "Route ok", target, metadata) - for _, q := range s.recorder.Get() { + // The mux records every handled request on the status page; the metadata the + // handler was given must not carry the secret into it either. + rec := extproc.NewQueryRecorder(10) + rec.AddRouterRequest(time.Now(), time.Millisecond, "Route ok", res.Target, md) + for _, q := range rec.Get() { if blob, _ := json.Marshal(q); strings.Contains(string(blob), secret) { t.Errorf("recorder/statusz retained sensitive value: %s", blob) } } } -func TestExtProcHeadersEvaluation(t *testing.T) { +func TestHandleRequestHeaders(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" tests := []struct { @@ -195,20 +199,17 @@ func TestExtProcHeadersEvaluation(t *testing.T) { // Parking disabled: these cases assert fail-fast mapping of resume // errors (e.g. FailedPrecondition -> immediate 503). Parking behavior - // is covered separately in TestExtProc_ParkingLotFull and resumer_test.go. - s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{}, nil, false, nil) + // is covered separately in TestHandleRequestHeaders_ParkingLotFull and + // resumer_test.go. + h := New(clientMock, ParkedRequestConfig{}, nil, false) - reqHeaders := &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":path", Value: "/v1/actors/invoke"}, - {Key: ":authority", Value: tc.authority}, - {Key: ":method", Value: "POST"}, - }, - }, - } + md := requestMetadata( + &corev3.HeaderValue{Key: ":path", Value: "/v1/actors/invoke"}, + &corev3.HeaderValue{Key: ":authority", Value: tc.authority}, + &corev3.HeaderValue{Key: ":method", Value: "POST"}, + ) - res, metadata, target, _, _, _, err := s.handleRequestHeaders(context.Background(), reqHeaders) + res, err := h.HandleRequestHeaders(context.Background(), md) if tc.expectErr { if err == nil { t.Fatalf("expected error but got nil") @@ -216,11 +217,11 @@ func TestExtProcHeadersEvaluation(t *testing.T) { if tc.expectedErrStr != "" && err.Error() != tc.expectedErrStr { t.Errorf("client body mismatch:\n got: %q\n want: %q", err.Error(), tc.expectedErrStr) } - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("expected *reqError, got %T (%v)", err, err) + t.Fatalf("expected *extproc.ReqError, got %T (%v)", err, err) } - if got, want := reqErr.statusCode, int(tc.expectedStatus); got != want { + if got, want := reqErr.StatusCode, int(tc.expectedStatus); got != want { t.Errorf("HTTP status code = %d, want %d", got, want) } if tc.resumeErr != nil && !errors.Is(err, tc.resumeErr) { @@ -232,11 +233,11 @@ func TestExtProcHeadersEvaluation(t *testing.T) { if err != nil { t.Fatalf("ext_proc processing error: %v", err) } - if target != tc.expectedTarget { - t.Errorf("expected target %q, got %q", tc.expectedTarget, target) + if res.Target != tc.expectedTarget { + t.Errorf("expected target %q, got %q", tc.expectedTarget, res.Target) } - mutation := res.Response.GetHeaderMutation() + mutation := res.Response.GetResponse().GetHeaderMutation() if len(mutation.GetSetHeaders()) != 2 { t.Fatalf("expected exactly two header options, found: %v", mutation.GetSetHeaders()) } @@ -252,19 +253,13 @@ func TestExtProcHeadersEvaluation(t *testing.T) { t.Errorf("original host mutation = %q, want %q", got, tc.authority) } - // Confirm that query logs recorded metric trace details - s.recorder.AddRouterRequest(time.Now(), 10*time.Millisecond, "Route ok", tc.expectedTarget, metadata) - queries := s.recorder.Get() - if len(queries) != 1 { - t.Errorf("expected query trace entries, got: %v", queries) - } }) } } -// TestExtProc_ParkingLotFull verifies that when the parking lot is at capacity +// TestHandleRequestHeaders_ParkingLotFull verifies that when the parking lot is at capacity // the request is shed with a 503 before any resume is attempted. -func TestExtProc_ParkingLotFull(t *testing.T) { +func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" var resumeCalled bool @@ -277,31 +272,27 @@ func TestExtProc_ParkingLotFull(t *testing.T) { // A 1-slot lot with the slot already occupied deterministically simulates a // full lot without needing a concurrent in-flight request. - s := NewExtProcServer(50051, clientMock, nil, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false, nil) - release, ok := s.parking.enter(context.Background()) + h := New(clientMock, ParkedRequestConfig{Budget: time.Second, Max: 1}, nil, false) + release, ok := h.parking.enter(context.Background()) if !ok { t.Fatal("priming enter should be admitted") } defer release(parkOutcomeServed) - reqHeaders := &extprocv3.HttpHeaders{ - Headers: &corev3.HeaderMap{ - Headers: []*corev3.HeaderValue{ - {Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, - }, - }, - } + md := requestMetadata( + &corev3.HeaderValue{Key: ":authority", Value: testUUID + ".team-a.actors.resources.substrate.ate.dev"}, + ) - _, _, _, _, _, _, err := s.handleRequestHeaders(context.Background(), reqHeaders) + _, err := h.HandleRequestHeaders(context.Background(), md) if err == nil { t.Fatal("expected error when parking lot is full") } - var reqErr *reqError + var reqErr *extproc.ReqError if !errors.As(err, &reqErr) { - t.Fatalf("expected *reqError, got %T (%v)", err, err) + t.Fatalf("expected *extproc.ReqError, got %T (%v)", err, err) } - if reqErr.statusCode != int(envoy_type.StatusCode_ServiceUnavailable) { - t.Errorf("status code = %d, want %d (503)", reqErr.statusCode, envoy_type.StatusCode_ServiceUnavailable) + if reqErr.StatusCode != int(envoy_type.StatusCode_ServiceUnavailable) { + t.Errorf("status code = %d, want %d (503)", reqErr.StatusCode, envoy_type.StatusCode_ServiceUnavailable) } if !strings.Contains(reqErr.Error(), "router at capacity") { t.Errorf("error body = %q, want it to mention capacity", reqErr.Error()) @@ -311,117 +302,6 @@ func TestExtProc_ParkingLotFull(t *testing.T) { } } -func TestClassifyOutcome(t *testing.T) { - tests := []struct { - name string - err error - expected string - }{ - { - name: "nil error maps to ok", - err: nil, - expected: "ok", - }, - { - name: "context Canceled maps to cancelled", - err: context.Canceled, - expected: "cancelled", - }, - { - name: "context DeadlineExceeded maps to timeout", - err: context.DeadlineExceeded, - expected: "timeout", - }, - { - name: "FailedPrecondition gRPC code maps to no_capacity", - err: status.Error(codes.FailedPrecondition, "capacity full"), - expected: "no_capacity", - }, - { - name: "Aborted gRPC code maps to lock_conflict", - err: status.Error(codes.Aborted, "lock conflict"), - expected: "lock_conflict", - }, - { - name: "NotFound gRPC code maps to not_found", - err: status.Error(codes.NotFound, "missing"), - expected: "not_found", - }, - { - name: "Unavailable gRPC code maps to unavailable", - err: status.Error(codes.Unavailable, "control-plane down"), - expected: "unavailable", - }, - { - name: "ResourceExhausted gRPC code maps to rate_limited", - err: status.Error(codes.ResourceExhausted, "rate limit exceeded"), - expected: "rate_limited", - }, - { - name: "StatusCode_NotFound reqError maps to not_found", - err: newReqError(envoy_type.StatusCode_NotFound, "missing"), - expected: "not_found", - }, - { - name: "StatusCode_ServiceUnavailable reqError maps to no_capacity", - err: newReqError(envoy_type.StatusCode_ServiceUnavailable, "no free workers"), - expected: "no_capacity", - }, - { - name: "StatusCode_TooManyRequests reqError maps to rate_limited", - err: newReqError(envoy_type.StatusCode_TooManyRequests, "rate limited"), - expected: "rate_limited", - }, - { - name: "Unknown error maps to resume_error", - err: errors.New("internal storage glitch"), - expected: "resume_error", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := classifyOutcome(tc.err); got != tc.expected { - t.Errorf("classifyOutcome(%v) = %q, want %q", tc.err, got, tc.expected) - } - }) - } -} - -func TestRecordRouteDuration_Attributes(t *testing.T) { - reader := sdkmetric.NewManualReader() - mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) - h, err := mp.Meter("atenet-router").Float64Histogram(routeDurationMetricName) - if err != nil { - t.Fatalf("failed to create histogram: %v", err) - } - - s := NewExtProcServer(50051, nil, h, ParkedRequestConfig{}, nil, false, nil) - s.recordRouteDuration(context.Background(), 10*time.Millisecond, "team-a-ns", "tmpl-a", classifyOutcome(nil), string(ResumeOutcomeTriggered)) - - var rm metricdata.ResourceMetrics - if err := reader.Collect(context.Background(), &rm); err != nil { - t.Fatalf("Collect failed: %v", err) - } - - dp := rm.ScopeMetrics[0].Metrics[0].Data.(metricdata.Histogram[float64]).DataPoints[0] - wantAttrs := map[string]string{ - "ate.template.namespace": "team-a-ns", - "ate.template.name": "tmpl-a", - "ate.router.outcome": "ok", - "ate.router.resume": "triggered", - } - - for k, want := range wantAttrs { - val, exists := dp.Attributes.Value(attribute.Key(k)) - if !exists { - t.Errorf("missing metric attribute %q", k) - } else if val.AsString() != want { - t.Errorf("attribute %q = %q, want %q", k, val.AsString(), want) - } - } -} - func TestAddRoutingMutationsViaAuthority(t *testing.T) { mutation := &extprocv3.HeaderMutation{} addRoutingMutations("10.0.0.52:443", "actor-1.team-a.actors.resources.substrate.ate.dev", true, mutation) @@ -439,7 +319,7 @@ func TestAddRoutingMutationsViaAuthority(t *testing.T) { if got[strings.ToLower(atunnel.OriginalHostHeader)] != "actor-1.team-a.actors.resources.substrate.ate.dev" { t.Errorf("%s = %q", atunnel.OriginalHostHeader, got[strings.ToLower(atunnel.OriginalHostHeader)]) } - if got[authorityHeader] != "10.0.0.52:443" { - t.Errorf("%s = %q", authorityHeader, got[authorityHeader]) + if got[extproc.AuthorityHeader] != "10.0.0.52:443" { + t.Errorf("%s = %q", extproc.AuthorityHeader, got[extproc.AuthorityHeader]) } } diff --git a/cmd/atenet/internal/router/metrics.go b/cmd/atenet/internal/router/ingress/metrics.go similarity index 58% rename from cmd/atenet/internal/router/metrics.go rename to cmd/atenet/internal/router/ingress/metrics.go index 448ab3204..29bc26e70 100644 --- a/cmd/atenet/internal/router/metrics.go +++ b/cmd/atenet/internal/router/ingress/metrics.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -22,58 +22,33 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" ) +// Request-parking instruments. parking.active is the live count of parked +// requests; parking.wait.duration is how long each request stayed parked +// (labeled by outcome); parking.rejected counts requests shed because the +// parking lot was full. const ( - routerServiceName = "atenet-router" - - // atenet.router.route.duration measures the latency from when the ext_proc handler receives a request - // (Envoy -> EPP) until the target worker endpoint is resolved - routeDurationMetricName = "atenet.router.route.duration" - - // Request-parking instruments. parking.active is the live count of parked - // requests; parking.wait.duration is how long each request stayed parked - // (labeled by outcome); parking.rejected counts requests shed because the - // parking lot was full. parkingActiveMetricName = "atenet.router.parking.active" parkingWaitMetricName = "atenet.router.parking.wait.duration" parkingRejectedMetricName = "atenet.router.parking.rejected" ) -// newRouteDurationHistogram creates the atenet.router.route.duration histogram from -// the global MeterProvider. -func newRouteDurationHistogram() (metric.Float64Histogram, error) { - h, err := otel.Meter(routerServiceName).Float64Histogram( - routeDurationMetricName, - metric.WithUnit("s"), - metric.WithDescription( - "latency between Substrate router receiving a request and resolving "+ - "the target worker endpoint, excluding actor execution and response", - ), - metric.WithExplicitBucketBoundaries( - 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, - 0.075, 0.1, 0.15, 0.2, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30, - ), - ) - if err != nil { - return nil, fmt.Errorf("create %s histogram: %w", routeDurationMetricName, err) - } - return h, nil -} - -// parkingMetrics bundles the OpenTelemetry instruments used by the parking lot. -// A nil *parkingMetrics is safe to use: every method becomes a no-op, which +// ParkingMetrics bundles the OpenTelemetry instruments used by the parking lot. +// A nil *ParkingMetrics is safe to use: every method becomes a no-op, which // keeps tests and metric-free deployments simple. -type parkingMetrics struct { +type ParkingMetrics struct { active metric.Int64UpDownCounter wait metric.Float64Histogram rejected metric.Int64Counter } -// newParkingMetrics creates the request-parking instruments from the global +// NewParkingMetrics creates the request-parking instruments from the global // MeterProvider. -func newParkingMetrics() (*parkingMetrics, error) { - meter := otel.Meter(routerServiceName) +func NewParkingMetrics() (*ParkingMetrics, error) { + meter := otel.Meter(extproc.ServiceName) active, err := meter.Int64UpDownCounter( parkingActiveMetricName, @@ -105,24 +80,24 @@ func newParkingMetrics() (*parkingMetrics, error) { return nil, fmt.Errorf("create %s counter: %w", parkingRejectedMetricName, err) } - return &parkingMetrics{active: active, wait: wait, rejected: rejected}, nil + return &ParkingMetrics{active: active, wait: wait, rejected: rejected}, nil } -func (m *parkingMetrics) addActive(ctx context.Context, delta int64) { +func (m *ParkingMetrics) addActive(ctx context.Context, delta int64) { if m == nil || m.active == nil { return } m.active.Add(ctx, delta) } -func (m *parkingMetrics) recordWait(ctx context.Context, d time.Duration, outcome parkOutcome) { +func (m *ParkingMetrics) recordWait(ctx context.Context, d time.Duration, outcome parkOutcome) { if m == nil || m.wait == nil { return } m.wait.Record(ctx, d.Seconds(), metric.WithAttributes(attribute.String("outcome", string(outcome)))) } -func (m *parkingMetrics) recordRejected(ctx context.Context) { +func (m *ParkingMetrics) recordRejected(ctx context.Context) { if m == nil || m.rejected == nil { return } diff --git a/cmd/atenet/internal/router/parking.go b/cmd/atenet/internal/router/ingress/parking.go similarity index 81% rename from cmd/atenet/internal/router/parking.go rename to cmd/atenet/internal/router/ingress/parking.go index fa0265366..a00f66e9a 100644 --- a/cmd/atenet/internal/router/parking.go +++ b/cmd/atenet/internal/router/ingress/parking.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -26,23 +26,31 @@ import ( // Default request-parking parameters. See ParkedRequestConfig for the meaning of each // field; these are also the flag defaults wired up in NewRouterCmd. const ( - defaultParkedRequestBudget = 5 * time.Second + DefaultParkedRequestBudget = 5 * time.Second - // defaultParkedRequestMax is sized together with the ext_proc cluster's + // DefaultParkedRequestMax is sized together with the ext_proc cluster's // circuit breaker (--extproc-max-requests, derived as twice the lot by default): each parked // request holds one ext_proc stream, i.e. one active request against that // cluster, for its entire wait. Startup validation keeps an explicit breaker >= the // lot, and the default pair (1024 lot / 2048 breaker) leaves equal headroom // for the fast path. See buildCluster in xds.go. - defaultParkedRequestMax = 1024 + DefaultParkedRequestMax = 1024 // Retry cadence between resume attempts while a request is parked: a gentle // exponential backoff. - defaultParkedRequestRetryInterval = 100 * time.Millisecond - defaultParkedRequestRetryFactor = 1.1 - defaultParkedRequestRetryJitter = 0.1 + DefaultParkedRequestRetryInterval = 100 * time.Millisecond + DefaultParkedRequestRetryFactor = 1.1 + DefaultParkedRequestRetryJitter = 0.1 ) +// ParkingStatus is a snapshot of the request-parking lot for the status page. +type ParkingStatus struct { + Enabled bool `json:"enabled"` + Active int `json:"active"` + MaxParked int `json:"max_parked"` + MaxWait string `json:"max_wait"` +} + // parkOutcome is the terminal disposition of a parked request. It is recorded // as the `outcome` label on the parking.wait.duration histogram. type parkOutcome string @@ -81,31 +89,31 @@ type ParkedRequestConfig struct { RetryJitter float64 } -// enabled reports whether request parking is active. Parking has no separate +// Enabled reports whether request parking is active. Parking has no separate // on/off switch: setting Max to 0 disables it, applying a fail-fast behavior // (no admission cap, no retry on pool saturation). -func (c ParkedRequestConfig) enabled() bool { return c.Max > 0 } +func (c ParkedRequestConfig) Enabled() bool { return c.Max > 0 } -// normalized returns the config with non-positive budget and retry parameters +// Normalized returns the config with non-positive budget and retry parameters // replaced by their defaults, so every consumer (the resumer's retry loop and // the Envoy ext_proc timeout) sees the same effective values. -func (c ParkedRequestConfig) normalized() ParkedRequestConfig { +func (c ParkedRequestConfig) Normalized() ParkedRequestConfig { if c.Budget <= 0 { - c.Budget = defaultParkedRequestBudget + c.Budget = DefaultParkedRequestBudget } if c.RetryInterval <= 0 { - c.RetryInterval = defaultParkedRequestRetryInterval + c.RetryInterval = DefaultParkedRequestRetryInterval } if c.RetryFactor == 0 { - c.RetryFactor = defaultParkedRequestRetryFactor + c.RetryFactor = DefaultParkedRequestRetryFactor } return c } -// validate rejects retry parameters that would make parking misbehave rather +// Validate rejects retry parameters that would make parking misbehave rather // than merely differ: a factor below 1 shrinks delays toward zero and turns // the parked retry loop into a hot loop against the control plane. -func (c ParkedRequestConfig) validate() error { +func (c ParkedRequestConfig) Validate() error { if c.RetryFactor != 0 && c.RetryFactor < 1.0 { return fmt.Errorf("parked-request retry factor must be >= 1.0, got %v", c.RetryFactor) } @@ -115,15 +123,15 @@ func (c ParkedRequestConfig) validate() error { return nil } -// defaultParkedRequestConfig returns the built-in parking configuration +// DefaultParkedRequestConfig returns the built-in parking configuration // (matching the NewRouterCmd flag defaults). -func defaultParkedRequestConfig() ParkedRequestConfig { +func DefaultParkedRequestConfig() ParkedRequestConfig { return ParkedRequestConfig{ - Budget: defaultParkedRequestBudget, - Max: defaultParkedRequestMax, - RetryInterval: defaultParkedRequestRetryInterval, - RetryFactor: defaultParkedRequestRetryFactor, - RetryJitter: defaultParkedRequestRetryJitter, + Budget: DefaultParkedRequestBudget, + Max: DefaultParkedRequestMax, + RetryInterval: DefaultParkedRequestRetryInterval, + RetryFactor: DefaultParkedRequestRetryFactor, + RetryJitter: DefaultParkedRequestRetryJitter, } } @@ -136,13 +144,13 @@ func defaultParkedRequestConfig() ParkedRequestConfig { // accounting, applying the router's fail-fast behavior. type parkingLot struct { cfg ParkedRequestConfig - metrics *parkingMetrics + metrics *ParkingMetrics mu sync.Mutex active int // current number of occupied slots; guarded by mu } -func newParkingLot(cfg ParkedRequestConfig, m *parkingMetrics) *parkingLot { +func newParkingLot(cfg ParkedRequestConfig, m *ParkingMetrics) *parkingLot { return &parkingLot{cfg: cfg, metrics: m} } @@ -153,7 +161,7 @@ func newParkingLot(cfg ParkedRequestConfig, m *parkingMetrics) *parkingLot { // waiting. When parking is disabled every request is admitted and no slot // accounting or metrics are recorded. func (l *parkingLot) enter(ctx context.Context) (release func(outcome parkOutcome), ok bool) { - if !l.cfg.enabled() { + if !l.cfg.Enabled() { return func(parkOutcome) {}, true } @@ -198,7 +206,7 @@ func (l *parkingLot) activeCount() int { // status returns a snapshot of the lot for the /statusz page. func (l *parkingLot) status() ParkingStatus { return ParkingStatus{ - Enabled: l.cfg.enabled(), + Enabled: l.cfg.Enabled(), Active: l.activeCount(), MaxParked: l.cfg.Max, MaxWait: l.cfg.Budget.String(), diff --git a/cmd/atenet/internal/router/parking_test.go b/cmd/atenet/internal/router/ingress/parking_test.go similarity index 99% rename from cmd/atenet/internal/router/parking_test.go rename to cmd/atenet/internal/router/ingress/parking_test.go index 62f20f18d..ad8a529e6 100644 --- a/cmd/atenet/internal/router/parking_test.go +++ b/cmd/atenet/internal/router/ingress/parking_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" diff --git a/cmd/atenet/internal/router/resumer.go b/cmd/atenet/internal/router/ingress/resumer.go similarity index 96% rename from cmd/atenet/internal/router/resumer.go rename to cmd/atenet/internal/router/ingress/resumer.go index e6b01f705..ce4bd943f 100644 --- a/cmd/atenet/internal/router/resumer.go +++ b/cmd/atenet/internal/router/ingress/resumer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -20,6 +20,7 @@ import ( "sync/atomic" "time" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/ateattr" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -114,9 +115,9 @@ type resumerOption func(*ActorResumer) // resume is retried, at cfg's retry cadence, for up to cfg's budget. When // disabled, the resumer applies fail-fast-on-capacity behavior. func withParking(cfg ParkedRequestConfig) resumerOption { - cfg = cfg.normalized() + cfg = cfg.Normalized() return func(r *ActorResumer) { - r.parkEnabled = cfg.enabled() + r.parkEnabled = cfg.Enabled() if r.parkEnabled { r.budget = cfg.Budget } @@ -128,8 +129,8 @@ func NewActorResumer(apiClient ateapipb.ControlClient, opts ...resumerOption) *A r := &ActorResumer{ apiClient: apiClient, budget: failFastResumeBudget, - backoff: resumeBackoff(defaultParkedRequestRetryInterval, - defaultParkedRequestRetryFactor, defaultParkedRequestRetryJitter), + backoff: resumeBackoff(DefaultParkedRequestRetryInterval, + DefaultParkedRequestRetryFactor, DefaultParkedRequestRetryJitter), } for _, opt := range opts { opt(r) @@ -161,7 +162,7 @@ func (r *ActorResumer) retryable(err error) bool { // requests within the process and, when parking is enabled, holds the request // while retrying transient failures until the budget elapses. func (r *ActorResumer) ResumeActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, ResumeOutcome, error) { - ctx, span := otel.Tracer(routerServiceName).Start(ctx, "ResumeActor", + ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ResumeActor", trace.WithAttributes(ateattr.ActorRefAttributes(actorRef)...)) defer span.End() diff --git a/cmd/atenet/internal/router/resumer_test.go b/cmd/atenet/internal/router/ingress/resumer_test.go similarity index 99% rename from cmd/atenet/internal/router/resumer_test.go rename to cmd/atenet/internal/router/ingress/resumer_test.go index b632789a1..225e59257 100644 --- a/cmd/atenet/internal/router/resumer_test.go +++ b/cmd/atenet/internal/router/ingress/resumer_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package router +package ingress import ( "context" @@ -527,7 +527,7 @@ func TestResumeBackoffHasNoCap(t *testing.T) { // Steps the moment the delay reaches Cap, which would end parking retries far // short of the budget (a 2s Cap stops the loop in ~7 steps / ~5s). The budget // context — not the step count or a cap — must bound how long a request parks. - b := resumeBackoff(defaultParkedRequestRetryInterval, defaultParkedRequestRetryFactor, defaultParkedRequestRetryJitter) + b := resumeBackoff(DefaultParkedRequestRetryInterval, DefaultParkedRequestRetryFactor, DefaultParkedRequestRetryJitter) if b.Cap != 0 { t.Errorf("resume backoff must not set Cap (it would stop retries at the cap); got %v", b.Cap) } diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index 5a0298d64..f1cf98c35 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -22,6 +22,7 @@ import ( "log/slog" "net" "net/http" + "os" "os/signal" "syscall" @@ -38,6 +39,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/serverboot" v1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -61,51 +65,60 @@ func init() { type RouterServer struct { cfg routerConfig - Cmd *cobra.Command - k8sClient client.Client - clientset kubernetes.Interface - apiClient ateapipb.ControlClient - extprocSrv *ExtProcServer - health *routerHealth - atStore atStore + Cmd *cobra.Command + k8sClient client.Client + clientset kubernetes.Interface + apiClient ateapipb.ControlClient + // extprocSrv is the ext_proc mux. Which handlers it carries — ingress, + // egress, or both — follows cfg.Mode. + extprocSrv *extproc.Server + // ingressHandler is the ingress handler registered on extprocSrv, kept for + // the status page's parking snapshot. Nil in egress-only mode. + ingressHandler *ingress.Handler + health *routerHealth + atStore atStore } func NewRouterServer(cfg routerConfig) (*RouterServer, error) { var k8sClient client.Client var clientset kubernetes.Interface + var store atStore - if cfg.TemplatesFile == "" { - k8sCfg, err := config.GetConfig() - if err != nil { - if cfg.Kubeconfig != "" { - k8sCfg, err = clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) - if err != nil { - return nil, fmt.Errorf("failed to read config from path %s: %w", cfg.Kubeconfig, err) + // Only ingress needs Kubernetes: it is the ActorTemplate controller and the + // xDS server that read from it. An egress-only instance is pure ext_proc and + // deliberately runs without any cluster access at all, so do not even build + // the clients — in-cluster config would only fail for want of RBAC. + if cfg.Mode.ServesIngress() { + if cfg.TemplatesFile == "" { + k8sCfg, err := config.GetConfig() + if err != nil { + if cfg.Kubeconfig != "" { + k8sCfg, err = clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to read config from path %s: %w", cfg.Kubeconfig, err) + } + } else { + return nil, fmt.Errorf("unable to establish Kubernetes configuration parameters: %w", err) } - } else { - return nil, fmt.Errorf("unable to establish Kubernetes configuration parameters: %w", err) } - } - slog.Info("Connecting to Kubernetes API server", slog.String("host", k8sCfg.Host)) + slog.Info("Connecting to Kubernetes API server", slog.String("host", k8sCfg.Host)) - k8sClient, err = client.New(k8sCfg, client.Options{ - Scheme: scheme, - }) - if err != nil { - return nil, fmt.Errorf("failed to initialize cluster client: %w", err) - } + k8sClient, err = client.New(k8sCfg, client.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize cluster client: %w", err) + } - clientset, err = kubernetes.NewForConfig(k8sCfg) - if err != nil { - return nil, fmt.Errorf("failed to initialize core client: %w", err) - } - } + clientset, err = kubernetes.NewForConfig(k8sCfg) + if err != nil { + return nil, fmt.Errorf("failed to initialize core client: %w", err) + } - var store atStore - if cfg.TemplatesFile != "" { - store = newFileATStore(cfg.TemplatesFile) - } else { - store = newk8sATStore(k8sClient) + store = newk8sATStore(k8sClient) + } else { + store = newFileATStore(cfg.TemplatesFile) + } } return &RouterServer{ @@ -137,7 +150,7 @@ func (s *RouterServer) Run(ctx context.Context) error { if err := s.cfg.validate(); err != nil { return fmt.Errorf("invalid router configuration: %w", err) } - parkCfg := s.cfg.ParkedRequest.normalized() + parkCfg := s.cfg.ParkedRequest.Normalized() // The drain-complete marker persists container restarts (emptyDir); a // stale one would release the Envoy preStop hook the moment a later drain @@ -155,7 +168,7 @@ func (s *RouterServer) Run(ctx context.Context) error { // sampler and Envoy's RandomSampling percent cannot drift. sampling := serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(dataPlaneTraceRatio)) tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{ - ServiceName: routerServiceName, + ServiceName: extproc.ServiceName, Sampling: sampling, }) if err != nil { @@ -163,7 +176,7 @@ func (s *RouterServer) Run(ctx context.Context) error { } defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown) - mp, err := serverboot.InitMetrics(ctx, routerServiceName) + mp, err := serverboot.InitMetrics(ctx, extproc.ServiceName) if err != nil { return fmt.Errorf("failed to initialize metrics: %w", err) } @@ -202,40 +215,62 @@ func (s *RouterServer) Run(ctx context.Context) error { s.apiClient = ateapipb.NewControlClient(conn) slog.InfoContext(ctx, "Starting substrate router subsystem", + slog.String("mode", string(s.cfg.Mode)), slog.Bool("standalone", s.cfg.Standalone), slog.String("atenet_router", string(s.cfg.atenetRouter()))) g, ctx := errgroup.WithContext(ctx) - if s.extprocSrv == nil { - routeDuration, err := newRouteDurationHistogram() - if err != nil { - return fmt.Errorf("failed to create route-duration histogram: %w", err) - } - parkMetrics, err := newParkingMetrics() + // Register one handler per direction this instance serves. The mux refuses + // any direction missing from this map, so the mode is enforced here rather + // than merely advertised. + handlers := extproc.Handlers{} + if s.cfg.Mode.ServesIngress() { + parkMetrics, err := ingress.NewParkingMetrics() if err != nil { return fmt.Errorf("failed to create parking metrics: %w", err) } + s.ingressHandler = ingress.New(s.apiClient, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority()) + handlers[s.ingressHandler.Direction()] = s.ingressHandler + } + if s.cfg.Mode.ServesEgress() { // Load the actor-identity CA up front so a missing or unusable bundle // fails startup, rather than turning into a 503 on the first actor - // egress attempt. An unset flag is the ingress-only case and is fine. + // egress attempt. An unset flag leaves the handler with no roots, which + // denies every CONNECT — see egress.New. var actorIdentityRoots *x509.CertPool if s.cfg.ActorIdentityCAFile != "" { pemBytes, err := os.ReadFile(s.cfg.ActorIdentityCAFile) if err != nil { return fmt.Errorf("reading --actor-identity-ca-file: %w", err) } - actorIdentityRoots, err = loadActorIdentityRoots(pemBytes) + actorIdentityRoots, err = egress.LoadActorIdentityRoots(pemBytes) if err != nil { return fmt.Errorf("loading --actor-identity-ca-file %q: %w", s.cfg.ActorIdentityCAFile, err) } } - s.extprocSrv = NewExtProcServer(s.cfg.ExtprocPort, s.apiClient, routeDuration, parkCfg, parkMetrics, s.cfg.atenetRouter().routeViaAuthority(), actorIdentityRoots) + egressHandler := egress.New(s.apiClient, actorIdentityRoots) + handlers[egressHandler.Direction()] = egressHandler } + + if s.extprocSrv == nil { + routeDuration, err := extproc.NewRouteDurationHistogram() + if err != nil { + return fmt.Errorf("failed to create route-duration histogram: %w", err) + } + s.extprocSrv = extproc.NewServer(s.cfg.ExtprocPort, routeDuration, handlers) + } + s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg) - if err := s.startDataplane(ctx, g, parkCfg, sampling.RootSamplingPercent()); err != nil { - return err + // The dataplane control plane — the xDS server and the ActorTemplate + // controller — configures the *ingress* Envoy. The egress Envoy is + // statically configured, so an egress-only instance runs neither and needs + // no Kubernetes access. + if s.cfg.Mode.ServesIngress() { + if err := s.startDataplane(ctx, g, parkCfg, sampling.RootSamplingPercent()); err != nil { + return err + } } // Start periodic service checking logic diff --git a/cmd/atenet/internal/router/status.go b/cmd/atenet/internal/router/status.go index 6a7625373..6c950a0cc 100644 --- a/cmd/atenet/internal/router/status.go +++ b/cmd/atenet/internal/router/status.go @@ -25,137 +25,38 @@ import ( "os" "runtime/debug" "strings" - "sync" "time" "github.com/spf13/pflag" "go.opentelemetry.io/otel" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) var BuildTag = "dev" -type RecordedQuery struct { - Timestamp time.Time `json:"timestamp"` - Client string `json:"client"` - Host string `json:"host"` - Path string `json:"path"` - Method string `json:"method"` - Action string `json:"action"` - Target string `json:"target"` - Duration time.Duration `json:"duration"` -} - -type QueryRecorder struct { - mu sync.RWMutex - queries []RecordedQuery - size int - index int -} - -func NewQueryRecorder(size int) *QueryRecorder { - return &QueryRecorder{ - queries: make([]RecordedQuery, 0, size), - size: size, - } -} - -func (qr *QueryRecorder) Add(q RecordedQuery) { - if qr == nil { - return - } - - qr.mu.Lock() - defer qr.mu.Unlock() - - if len(qr.queries) < qr.size { - qr.queries = append(qr.queries, q) - } else { - qr.queries[qr.index] = q - qr.index = (qr.index + 1) % qr.size - } -} - -func (qr *QueryRecorder) Get() []RecordedQuery { - if qr == nil { - return nil - } - - qr.mu.RLock() - defer qr.mu.RUnlock() - - n := len(qr.queries) - if n == 0 { - return nil - } - - res := make([]RecordedQuery, n) - if n < qr.size { - for i := 0; i < n; i++ { - res[i] = qr.queries[n-1-i] - } - } else { - for i := 0; i < n; i++ { - pos := (qr.index - 1 - i + n) % n - res[i] = qr.queries[pos] - } - } - - return res -} - -// redactPath drops the query string, which may carry credentials (CWE-598). -func redactPath(path string) string { - p, _, _ := strings.Cut(path, "?") - return p -} - -func (qr *QueryRecorder) AddRouterRequest( - start time.Time, - duration time.Duration, - action, - target string, - m *requestMetadata, -) { - qr.Add(RecordedQuery{ - Timestamp: start, - Client: m.headers[authorityHeader], - Host: m.host, - Path: redactPath(m.path), - Method: m.headers[":method"], - Action: action, - Target: target, - Duration: duration, - }) -} - type TemplateInfo struct { Name string `json:"name"` Namespace string `json:"namespace"` } -// ParkingStatus is a snapshot of the request-parking lot for the status page. -type ParkingStatus struct { - Enabled bool `json:"enabled"` - Active int `json:"active"` - MaxParked int `json:"max_parked"` - MaxWait string `json:"max_wait"` -} - type DashboardContext struct { - BuildTag string `json:"build_tag"` - RouterClusterIP string `json:"router_cluster_ip"` - Namespace string `json:"namespace"` - HttpPort int `json:"port_http"` - XdsPort int `json:"port_xds"` - ExtprocPort int `json:"port_extproc"` - StatusPort int `json:"status_port"` - Args string `json:"args"` - Flags map[string]string `json:"flags"` - Queries []FormattedQuery `json:"queries"` - Health RouterHealthReport `json:"health"` - Templates []TemplateInfo `json:"templates"` - Parking ParkingStatus `json:"parking"` + BuildTag string `json:"build_tag"` + RouterClusterIP string `json:"router_cluster_ip"` + Namespace string `json:"namespace"` + Mode Mode `json:"mode"` + HttpPort int `json:"port_http"` + XdsPort int `json:"port_xds"` + ExtprocPort int `json:"port_extproc"` + StatusPort int `json:"status_port"` + Args string `json:"args"` + Flags map[string]string `json:"flags"` + Queries []FormattedQuery `json:"queries"` + Health RouterHealthReport `json:"health"` + Templates []TemplateInfo `json:"templates"` + Parking ingress.ParkingStatus `json:"parking"` } type FormattedQuery struct { @@ -187,7 +88,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string { } func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { - ctx, span := otel.Tracer(routerServiceName).Start(req.Context(), "handleStatusz") + ctx, span := otel.Tracer(extproc.ServiceName).Start(req.Context(), "handleStatusz") defer span.End() ctx, cancel := context.WithTimeout(ctx, 3*time.Second) @@ -213,9 +114,9 @@ func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { }) } - var rawQueries []RecordedQuery - if s.extprocSrv != nil && s.extprocSrv.recorder != nil { - rawQueries = s.extprocSrv.recorder.Get() + var rawQueries []extproc.RecordedQuery + if s.extprocSrv != nil { + rawQueries = s.extprocSrv.Queries() } formattedQueries := make([]FormattedQuery, len(rawQueries)) @@ -253,15 +154,17 @@ func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) { } } - var parking ParkingStatus - if s.extprocSrv != nil { - parking = s.extprocSrv.parking.status() + // Parking belongs to the ingress handler; an egress-only instance has none. + var parking ingress.ParkingStatus + if s.ingressHandler != nil { + parking = s.ingressHandler.ParkingStatus() } data := DashboardContext{ BuildTag: buildInfo, RouterClusterIP: routerIP, Namespace: s.cfg.Namespace, + Mode: s.cfg.Mode, HttpPort: s.cfg.HttpPort, XdsPort: s.cfg.XdsPort, ExtprocPort: s.cfg.ExtprocPort, diff --git a/cmd/atenet/internal/router/status_test.go b/cmd/atenet/internal/router/status_test.go index 7f5234097..5be288f97 100644 --- a/cmd/atenet/internal/router/status_test.go +++ b/cmd/atenet/internal/router/status_test.go @@ -33,6 +33,9 @@ import ( "strings" "testing" "time" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func TestStatuszEndpoint(t *testing.T) { @@ -68,6 +71,7 @@ func TestStatuszEndpoint(t *testing.T) { ExtProcMaxRequests: defaultExtProcMaxRequests, TemplatesFile: tmpFile.Name(), MetricsAddr: "127.0.0.1:0", + ParkedRequest: ingress.DefaultParkedRequestConfig(), Auth: authConfig{ AteapiCAFile: caPath, AteapiClientCertPath: clientCertPath, @@ -79,13 +83,13 @@ func TestStatuszEndpoint(t *testing.T) { t.Fatalf("Failed generating router server: %v", err) } - srv.extprocSrv = NewExtProcServer(cfg.ExtprocPort, &mockClient{}, nil, defaultParkedRequestConfig(), nil, false, nil) + srv.extprocSrv = extproc.NewServer(cfg.ExtprocPort, nil, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Inject recorded queries - srv.extprocSrv.recorder.Add(RecordedQuery{ + srv.extprocSrv.Recorder().Add(extproc.RecordedQuery{ Timestamp: time.Now(), Client: "127.0.0.1", Host: "example.com", @@ -164,8 +168,8 @@ func TestStatuszEndpoint(t *testing.T) { if !dashboard.Parking.Enabled { t.Errorf("expected parking reported as enabled in status JSON") } - if dashboard.Parking.MaxParked != defaultParkedRequestMax { - t.Errorf("expected parking max_parked %d, got %d", defaultParkedRequestMax, dashboard.Parking.MaxParked) + if dashboard.Parking.MaxParked != ingress.DefaultParkedRequestMax { + t.Errorf("expected parking max_parked %d, got %d", ingress.DefaultParkedRequestMax, dashboard.Parking.MaxParked) } } diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index bcf07c7e6..d2ec58dec 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -58,6 +58,8 @@ import ( cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" serverv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) const ( @@ -76,12 +78,10 @@ const ( httpProtocolOptionsName = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" // OriginalDstClusterName routes actor traffic to the worker's atunnel - // ingress by the IP:port the ext_proc puts in OriginalDstHeader, while the - // request :authority stays the actor DNS name so atunnel can identify the - // active actor. + // ingress by the IP:port the ext_proc puts in ingress.OriginalDstHeader, + // while the request :authority stays the actor DNS name so atunnel can + // identify the active actor. OriginalDstClusterName = "actor_original_dst" - // OriginalDstHeader carries the resolved worker atunnel address (IP:443). - OriginalDstHeader = "x-ate-original-dst" ) // defaultExtProcMessageTimeout is Envoy's per-message ext_proc response timeout @@ -90,7 +90,7 @@ const ( const defaultExtProcMessageTimeout = 5 * time.Second // defaultExtProcMaxRequests is the circuit-breaker max_requests set on the -// ext_proc cluster: defaultParkedRequestMax plus equal fast-path headroom, so a +// ext_proc cluster: ingress.DefaultParkedRequestMax plus equal fast-path headroom, so a // full parking lot cannot starve the millisecond-scale header exchanges of // requests to already-running actors. See buildCluster. const defaultExtProcMaxRequests = 2048 @@ -621,7 +621,7 @@ func (x *XdsServer) buildUpstreamTransportSocket() *corev3.TransportSocket { } // buildOriginalDstCluster dials the exact worker atunnel address supplied by -// the ext_proc in OriginalDstHeader. Unlike the dynamic_forward_proxy cluster, +// the ext_proc in ingress.OriginalDstHeader. Unlike the dynamic_forward_proxy cluster, // it does not derive the destination from :authority, so the request keeps the // actor DNS name as its Host for atunnel to authorize. mTLS to atunnel is // applied via the shared upstream transport socket (SPIFFE URI validation). @@ -636,7 +636,7 @@ func (x *XdsServer) buildOriginalDstCluster() *clusterv3.Cluster { LbConfig: &clusterv3.Cluster_OriginalDstLbConfig_{ OriginalDstLbConfig: &clusterv3.Cluster_OriginalDstLbConfig{ UseHttpHeader: true, - HttpHeaderName: OriginalDstHeader, + HttpHeaderName: ingress.OriginalDstHeader, }, }, } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 3b4e6aef0..98287eae8 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -43,6 +43,8 @@ import ( secretgrpc "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" ) func TestXdsServer_UpdateSnapshot(t *testing.T) { @@ -540,8 +542,8 @@ func TestXdsServer_ExtProcCircuitBreaker(t *testing.T) { if got != uint32(defaultExtProcMaxRequests) { t.Errorf("default max_requests = %d, want %d", got, defaultExtProcMaxRequests) } - if got < uint32(defaultParkedRequestMax) { - t.Errorf("default breaker (%d) below the default lot (%d): a full lot would be truncated by Envoy", got, defaultParkedRequestMax) + if got < uint32(ingress.DefaultParkedRequestMax) { + t.Errorf("default breaker (%d) below the default lot (%d): a full lot would be truncated by Envoy", got, ingress.DefaultParkedRequestMax) } }) diff --git a/demos/egress/README.md b/demos/egress/README.md index 9360b40c1..763aac7e7 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -44,18 +44,20 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r 4. **Identity authentication.** Envoy requires a client certificate signed by the actor-identity CA, so a non-actor client is refused at the handshake. It then forwards the verified chain to `ext_proc` as `x-forwarded-client-cert`, and the **atenet router** (co-located in the gateway - pod as an ext_proc sidecar, the same ext_proc code used for ingress) re-verifies the chain, - requires exactly one `ActorIdentity` extension with `purpose: atunnel`, and calls the ate API - (`GetActor`). It returns **403** unless the certified **UID** matches a real, `RUNNING` actor. - This mirrors the ingress gateway's Envoy + ext_proc co-location; a standalone/shared ext_proc - is a future step. + pod as an ext_proc sidecar, the same binary that serves ingress, started with `--mode=egress`) + re-verifies the chain, requires exactly one `ActorIdentity` extension with `purpose: atunnel`, + and calls the ate API (`GetActor`). It returns **403** unless the certified **UID** matches a + real, `RUNNING` actor. This mirrors the ingress gateway's Envoy + ext_proc co-location; a + standalone/shared ext_proc is a future step. ## Components - **Egress app (`main.go`)** — the Actor: `POST /` with `{"url":"..."}` → fetches it → returns status + body. - **Egress gateway** — `manifests/ate-install/atenet-egress.yaml`. One pod, two containers: - an Envoy (`envoy`) and the atenet router ext_proc (`ext-proc`), called over localhost. + an Envoy (`envoy`) and the atenet router ext_proc (`ext-proc`, `--mode=egress`), called over + localhost. In egress mode the router serves the egress ext_proc handler only — no xDS server, + no ActorTemplate controller, and no Kubernetes access at all. - **Egress opt-in** — `ate-api-server --egress-gateway-address=atenet-egress.ate-system.svc:443` (set in `manifests/ate-install/ate-api-server.yaml`). ateapi stamps the address onto every atelet `Run`/`Restore`, which turns on tunneled egress cluster-wide. diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 924c440aa..c395d2a20 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -29,35 +29,11 @@ kind: ServiceAccount metadata: name: atenet-egress namespace: ate-system ---- -# RBAC for the co-located ext_proc sidecar (atenet router), which watches -# ActorTemplates as part of its normal operation. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: atenet-egress -rules: -- apiGroups: - - "ate.dev" - resources: - - actortemplates - verbs: - - get - - watch - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: atenet-egress -subjects: -- kind: ServiceAccount - name: atenet-egress - namespace: ate-system -roleRef: - kind: ClusterRole - name: atenet-egress - apiGroup: rbac.authorization.k8s.io +# +# No RBAC is bound to this ServiceAccount on purpose: the ext_proc sidecar runs +# with --mode=egress, which serves the egress ext_proc handler and nothing else +# — no xDS server and no ActorTemplate controller, so no Kubernetes access. It +# reaches the control plane over the ate API only. --- apiVersion: v1 kind: ConfigMap @@ -77,7 +53,7 @@ data: filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in - # cmd/atenet/internal/router/extproc_egress.go. + # cmd/atenet/internal/router/extproc/dispatch.go. - name: egress transport_socket: name: envoy.transport_sockets.tls @@ -173,9 +149,10 @@ data: # this Envoy-asserted filter chain name so that no client can # select the egress path by crafting a request. The value must # match EgressFilterChainName in - # cmd/atenet/internal/router/extproc_egress.go; renaming the - # filter chain above without updating it fails closed (egress - # requests take the ingress path and 404). + # cmd/atenet/internal/router/extproc/dispatch.go; renaming the + # filter chain above without updating it fails closed (the + # request is classified as ingress, which this --mode=egress + # instance does not serve, and is refused with a 404). request_attributes: - xds.filter_chain_name processing_mode: @@ -298,15 +275,20 @@ spec: - name: actor-id-ca-certs mountPath: /run/actor-id-ca-certs readOnly: true - # Co-located ext_proc server (the atenet router, ext_proc-only). The egress - # Envoy calls it over localhost to authenticate actor identity against the - # ate API on every CONNECT. This mirrors the ingress gateway topology - # (Envoy + ext_proc in one pod); a shared/standalone ext_proc is a future step. + # Co-located ext_proc server: the same atenet router binary the ingress + # gateway runs, started with --mode=egress so it serves only the egress + # ext_proc handler (no xDS server, no ActorTemplate controller, no + # Kubernetes access). The egress Envoy calls it over localhost to + # authenticate actor identity against the ate API on every CONNECT. + # + # Ingress and egress are separate Deployments because they scale + # independently, not because they need separate binaries: one instance can + # serve both directions with --mode=all. - name: ext-proc image: ko://github.com/agent-substrate/substrate/cmd/atenet args: - router - - --standalone + - --mode=egress - --namespace=ate-system - --port-extproc=50051 - --extproc-address=127.0.0.1 diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index f29b6fd35..e794e5391 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -161,6 +161,12 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atenet args: - "router" + # Serve the ingress direction only: the xDS server and ActorTemplate + # controller for the Envoy in this pod, plus the ingress ext_proc + # handler. Egress runs the same binary with --mode=egress in its own + # Deployment (manifests/ate-install/atenet-egress.yaml), because the two + # directions scale independently. + - "--mode=ingress" - "--standalone" - "--namespace=ate-system" - "--port-http=8080" diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index 5ce179141..107c8950f 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -34,15 +34,15 @@ patches: namespace: ate-system patch: |- - op: test - path: /spec/template/spec/containers/0/args/4 + path: /spec/template/spec/containers/0/args/5 value: --port-xds=18000 - op: remove - path: /spec/template/spec/containers/0/args/4 + path: /spec/template/spec/containers/0/args/5 - op: test - path: /spec/template/spec/containers/0/args/8 + path: /spec/template/spec/containers/0/args/9 value: --envoy-cert-path=/run/servicedns.podcert.ate.dev/credential-bundle.pem - op: remove - path: /spec/template/spec/containers/0/args/8 + path: /spec/template/spec/containers/0/args/9 - op: test path: /spec/template/spec/containers/0/ports/0/name value: xds diff --git a/manifests/ate-install/kind-token-client/kustomization.yaml b/manifests/ate-install/kind-token-client/kustomization.yaml index c7db1cc3a..d95c6be06 100644 --- a/manifests/ate-install/kind-token-client/kustomization.yaml +++ b/manifests/ate-install/kind-token-client/kustomization.yaml @@ -53,10 +53,10 @@ patches: namespace: ate-system patch: |- - op: test - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem - op: remove - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-use-token-auth=true diff --git a/manifests/ate-install/token-client/kustomization.yaml b/manifests/ate-install/token-client/kustomization.yaml index 3b172a17b..fc39fff8a 100644 --- a/manifests/ate-install/token-client/kustomization.yaml +++ b/manifests/ate-install/token-client/kustomization.yaml @@ -53,10 +53,10 @@ patches: namespace: ate-system patch: |- - op: test - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 value: --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem - op: remove - path: /spec/template/spec/containers/0/args/12 + path: /spec/template/spec/containers/0/args/13 - op: add path: /spec/template/spec/containers/0/args/- value: --ateapi-use-token-auth=true From ba898ea71a2e3cdae190ca0ed98eab81681f6e12 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Thu, 6 Aug 2026 15:48:59 -0700 Subject: [PATCH 5/6] egress: make verify-egress-demo.sh robust and executable The gateway access-log assertion read the log once, immediately after curl returned, and failed if it saw nothing new. That is racy twice over: * Envoy emits the CONNECT access-log entry asynchronously, so for an external destination it can land seconds after the actor's response. * The Actor's HTTP client keeps the tunnel alive. A repeat fetch to a host it already reached rides the open tunnel and produces no new entry at all, so a run against a warm actor failed even though egress was working. Poll for the new entry, and fall back to any tunnel already open for this actor's SAN before declaring failure. Also read -c envoy explicitly (the pod also runs the ext-proc sidecar) and mark the script executable, as every other directly-invoked script in hack/ is. --- cmd/atenet/internal/router/egress/egress.go | 33 +++++-------------- demos/egress/test-egress.sh | 7 ----- hack/install-ate.sh | 23 ++++---------- hack/verify-egress-demo.sh | 35 ++++++++++++++++----- internal/e2e/router_client_test.go | 8 ----- manifests/ate-install/ate-api-server.yaml | 5 +-- manifests/ate-install/atenet-egress.yaml | 34 +------------------- 7 files changed, 44 insertions(+), 101 deletions(-) mode change 100644 => 100755 hack/verify-egress-demo.sh diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 2031691fe..5e2a105fe 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -202,31 +202,14 @@ func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*su // actor certificate issued by the actor-identity CA, and returns the single // ActorIdentity it carries. // -// ########################################################################### -// SEE(lior): READ THIS IF YOU ARE WONDERING WHY WE VERIFY THE CHAIN TWICE. -// -// The egress listener already does full mTLS: require_client_certificate with -// the actor-identity CA as its trusted_ca, so Envoy refuses the handshake for -// anything this function would also reject on chain, expiry, or signature. The -// re-verification below is therefore redundant *today*, and it is here on -// purpose: -// -// - Envoy validates the chain but cannot look at the ActorIdentity -// extension. This function has to parse the certificate regardless, and -// parsing an unverified certificate and then trusting its contents is the -// failure mode that keeps producing CVEs. Verifying what we parse keeps the -// trust decision in one place instead of split across a YAML file and a Go -// file. -// - It makes the handler safe under Envoy config drift. Someone relaxing -// require_client_certificate, widening trusted_ca, or putting another proxy -// in front should not silently turn this into an unauthenticated endpoint. -// - It costs one signature verification per CONNECT, not per request: the -// tunnel is established once and then carries raw TCP. -// -// If you decide the Envoy-side check is authoritative and this is dead weight, -// this function is the thing to delete — but keep the ActorIdentity extraction -// and the IsCA/EKU/purpose checks below it, because Envoy does none of those. -// ########################################################################### +// The chain is verified here even though Envoy already did it at the handshake +// (require_client_certificate with the actor-identity CA as trusted_ca). We have +// to parse the certificate anyway to read the ActorIdentity extension, which +// Envoy cannot see, and trusting a parsed-but-unverified certificate is a +// well-worn source of CVEs. It also keeps the handler safe if the Envoy config +// is ever loosened, and costs one signature check per CONNECT rather than per +// request. The IsCA, ClientAuth-EKU, and purpose checks below have no Envoy-side +// equivalent at all. func (h *Handler) verifyActorCertificate(chain []*x509.Certificate) (*substratex509.ActorIdentity, error) { leaf := chain[0] intermediates := x509.NewCertPool() diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh index 7566146a1..271c550dd 100755 --- a/demos/egress/test-egress.sh +++ b/demos/egress/test-egress.sh @@ -124,13 +124,6 @@ fi ############################################################################## log "NEGATIVE — a pod identity is not an actor identity (expect a refused handshake)" ############################################################################## -# SEE(lior): this used to be a 403 assertion. On the header-based design the -# gateway trusted any podidentity holder's mTLS and let ext_proc adjudicate the -# X-Ate-* headers it asserted, so a probe pod could reach the CONNECT and be -# denied there. The gateway now trusts only the actor-identity CA, so the same -# probe never gets past the TLS handshake — the denial moved a layer down and -# there is no CONNECT response to read a status code out of. Assert the -# handshake failure instead; it is the stronger property. ${K} apply -f - >/dev/null <<'YAML' apiVersion: v1 kind: Pod diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 5154b1032..ca68115b1 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -280,24 +280,13 @@ create_actor_id_ca_pool_secret() { --secret-namespace=ate-system } -# --------------------------------------------------------------------------- -# SEE(lior): actor-identity CA trust bundle for the egress PEP. -# # The egress gateway has to verify actor client certificates, which means it -# needs the actor-identity CA *root* — and only the root. The authoritative -# copy today is the actor-id-ca-pool Secret, but its pool.json also carries the -# CA signing key, so mounting that Secret into the gateway would put the key -# that mints every actor identity inside a pod that only ever needs to verify -# them. This derives a cert-only Secret instead, following exactly the pattern -# create_valkey_ca_certs_secret already uses for the signer roots. +# needs the actor-identity CA root. actor-id-ca-pool Secret containts both +# root and CA signing key. This derives a cert-only Secret instead, following +# exactly the pattern create_valkey_ca_certs_secret already uses for the +# signer roots. # -# TODO(liorlieberman): revisit. The other CAs reach their consumers as -# ClusterTrustBundles published by the podcertificate controller, and the -# actor-identity CA arguably should too — then the gateway would project a -# trust bundle like it already does for servicedns/podidentity and this -# install-time Secret would go away. Doing that needs a signer/controller path -# that does not exist yet, so this is the interim shape. -# --------------------------------------------------------------------------- +# TODO(liorlieberman): should this be published as ClusterTrustBundles? create_actor_id_ca_certs_secret() { log_step "create_actor_id_ca_certs_secret" # Extract into its own variable first: errexit cannot see a substitution fail @@ -459,7 +448,7 @@ ensure_apiserver_prerequisites() { || create_jwt_authority_pool_secret run_kubectl get secret -n ate-system actor-id-ca-pool >/dev/null 2>&1 \ || create_actor_id_ca_pool_secret - # SEE(lior): derived from actor-id-ca-pool above, so it must come after it. + # Derived from actor-id-ca-pool above, so it must come after it. run_kubectl get secret -n ate-system actor-id-ca-certs >/dev/null 2>&1 \ || create_actor_id_ca_certs_secret run_kubectl get secret -n podcertificate-controller-system service-dns-ca-pool >/dev/null 2>&1 \ diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh old mode 100644 new mode 100755 index e1fadf543..abd641de2 --- a/hack/verify-egress-demo.sh +++ b/hack/verify-egress-demo.sh @@ -44,7 +44,9 @@ kubectl-ate --context "${CTX}" create actor "${ACTOR}" \ ${K} -n ate-system wait --for=condition=Ready "actor/${ACTOR}" 2>/dev/null || sleep 10 echo "== snapshot gateway log offset ==" -BEFORE=$(${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null | wc -l | tr -d ' ') +# -c envoy explicitly: the gateway pod also runs the ext-proc sidecar, and the +# [egress] access log belongs to Envoy. +BEFORE=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null | wc -l | tr -d ' ') echo "== drive actor egress: GET ${TARGET_URL} via the actor ==" ${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 & @@ -57,10 +59,29 @@ RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" echo "== NEW egress gateway access log lines (proof of CONNECT+mTLS+identity) ==" -${K} -n ate-system logs deployment/atenet-egress --tail=-1 2>/dev/null \ - | tail -n +"$((BEFORE + 1))" | grep '\[egress\]' || { - echo "!! no [egress] lines — dumping recent gateway logs:" - ${K} -n ate-system logs deployment/atenet-egress --tail=20 - exit 1 - } +# Envoy emits the CONNECT entry asynchronously (an external dst can land seconds +# after the actor's response), so we poll. And the Actor's HTTP client keeps the +# tunnel alive: a repeat fetch to a host it already reached rides the open tunnel +# and produces no new access-log entry at all, so a run against a warm actor +# would fail even though egress is working. Fall back to any tunnel already open +# for this actor's SAN before declaring failure. +SAN="atespace/${ATESPACE}/actor/${ACTOR}" +NEW="" +for _ in $(seq 1 15); do + NEW=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \ + | tail -n +"$((BEFORE + 1))" | grep '\[egress\]' || true) + [ -n "${NEW}" ] && break + sleep 2 +done +if [ -n "${NEW}" ]; then + echo "${NEW}" +elif ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \ + | grep -q "\[egress\].*${SAN}"; then + echo " no new CONNECT — the actor reused an already-open tunnel; its existing entries:" + ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 | grep "\[egress\].*${SAN}" | tail -3 +else + echo "!! no [egress] lines for ${SAN} — dumping recent gateway logs:" + ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=20 + exit 1 +fi echo "== PASS: actor egress traversed the Envoy egress gateway ==" diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go index d9be88241..e96351084 100644 --- a/internal/e2e/router_client_test.go +++ b/internal/e2e/router_client_test.go @@ -24,14 +24,6 @@ import ( "github.com/agent-substrate/substrate/internal/resources" ) -// SEE(lior): this file used to also hold TestResolveHTTPTargetPort and -// TestIsPodReady. Main moved both into internal/portforward/portforward_test.go -// (upstream 4453b5e7, "Prefactoring: Consolidate logic to port-forward to a -// service Pod") and deleted this file, so git rename-matched this branch's -// version of it against the new portforward test and reported the whole thing as -// a conflict. Took main's move as-is and re-added only the net-new PostJSON test -// here, rather than resurrecting the port-forward tests in a package that no -// longer owns that code. func TestRouterClientPostJSON(t *testing.T) { client := &RouterClient{ baseURL: "http://router.test", diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 6eced4b7a..c4c73a37d 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -104,14 +104,11 @@ spec: - --actor-id-ca-pool=/run/actor-id-ca-pool/pool.json - --atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem - --pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem - # ONLY UNTIL THE EGRESS API IS DECIDED, FOR POC: turn on pluggable actor + # TODO(lior):Remove when egress API pr is merged. This turns on pluggable actor # egress cluster-wide. ateapi stamps this address onto every atelet # Run/Restore, ateom hands it to atunnel, and actor TCP egress is # transparently redirected (nftables) into atunnel, which wraps it in # mTLS + HTTP CONNECT to the egress gateway. - # - # SEE(lior): this flag lived on atelet on the pre-rebase branch; PR 708 - # moved egress-gateway configuration to ateapi, so it is set here now. - --egress-gateway-address=atenet-egress.ate-system.svc:443 # Graceful shutdown knobs. The sum must fit within terminationGracePeriodSeconds. - --drain-delay=13s diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index c395d2a20..3776a8ea1 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -17,13 +17,6 @@ # actor-identity client cert), # * terminate the actor's HTTP CONNECT and tunnel raw TCP to the CONNECT # authority (the actor's original destination, always sent as IP:port). -# -# SEE(lior): the client cert is the actor's own identity — minted per actor by -# ateapi's actoridentity service off the actor-identity CA and carrying the -# ActorIdentity X.509 extension — not the pod's podidentity cert. The pod cert -# says "some substrate pod"; only the actor cert says *which actor*, which is -# the thing the gateway has to authorize. The CONNECT carries no identity -# headers at all; everything the PEP decides on comes out of the certificate. apiVersion: v1 kind: ServiceAccount metadata: @@ -89,9 +82,7 @@ data: # ActorIdentity in Go is to have Envoy hand over the raw chain. # SANITIZE_SET drops whatever x-forwarded-client-cert the client # sent and writes Envoy's own view of the verified peer, and - # chain: true puts the full URL-encoded PEM chain in it. Removing - # either of these breaks egress closed: the handler sees no XFCC - # and denies every CONNECT. + # chain: true puts the full URL-encoded PEM chain in it. forward_client_cert_details: SANITIZE_SET set_current_client_cert_details: chain: true @@ -106,16 +97,6 @@ data: "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog log_format: text_format_source: - # SEE(lior): the peer fields come from the verified client - # certificate, not from request headers. The actor sends no - # identity headers any more, and logging attacker-controlled - # ones would have made this log lie about who egressed. - # peer_san= is the actor's SPIFFE URI SAN, - # spiffe://substrate-actor.local/atespace//actor/ - # — the only actor-identifying field Envoy can format - # (ateapi mints these with an empty Subject, and the UID - # lives in the ActorIdentity extension Envoy cannot read). - # The ext_proc sidecar logs atespace/actor/UID separately. inline_string: "[egress] authority=%REQ(:AUTHORITY)% peer_san=%DOWNSTREAM_PEER_URI_SAN% peer_serial=%DOWNSTREAM_PEER_SERIAL% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% up_bytes=%BYTES_RECEIVED% down_bytes=%BYTES_SENT%\n" route_config: name: connect_route @@ -360,19 +341,6 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem - # SEE(lior): the actor-identity CA root, as a cert-only Secret that - # install-ate.sh derives from the actor-id-ca-pool Secret. It is a plain - # Secret rather than a projected clusterTrustBundle — unlike the two above - # — because the actor-identity CA has no signer/controller publishing a - # trust bundle for it yet; its only in-cluster home is actor-id-ca-pool, - # whose pool.json also holds the CA *signing key*. Mounting that here would - # put the key that mints every actor identity into the pod that merely - # verifies them, so install-ate.sh extracts the root and nothing else. - # - # TODO(liorlieberman): revisit once the actor-identity CA is published as a - # ClusterTrustBundle; this volume then becomes a third projected source and - # the install-time Secret goes away. See create_actor_id_ca_certs_secret in - # hack/install-ate.sh. - name: actor-id-ca-certs secret: secretName: actor-id-ca-certs From da323ec041e918e9de7204759fd34038496f1c8c Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Sun, 9 Aug 2026 18:59:02 -0700 Subject: [PATCH 6/6] fixes --- cmd/atenet/internal/router/README.md | 37 ++++---- cmd/atenet/internal/router/cmd.go | 12 +-- cmd/atenet/internal/router/config.go | 25 ++--- cmd/atenet/internal/router/config_test.go | 2 +- cmd/atenet/internal/router/egress/egress.go | 92 ++++++++++--------- .../internal/router/extproc/dispatch.go | 8 +- cmd/atenet/internal/router/extproc/errors.go | 4 +- cmd/atenet/internal/router/extproc/extproc.go | 27 +++--- cmd/atenet/internal/router/extproc/handler.go | 6 +- .../internal/router/extproc/metadata.go | 4 +- cmd/atenet/internal/router/extproc/metrics.go | 2 +- cmd/atenet/internal/router/health.go | 2 +- cmd/atenet/internal/router/ingress/ingress.go | 12 +-- cmd/atenet/internal/router/router.go | 18 ++-- demos/egress/README.md | 4 +- docs/dev/best-practices/otel-collector.md | 2 +- docs/roadmap.md | 2 +- hack/verify-egress-demo.sh | 3 +- internal/e2e/router_client.go | 2 +- internal/resources/actorref.go | 2 +- manifests/ate-install/atenet-egress.yaml | 43 +++++++-- manifests/ate-install/atenet-router.yaml | 8 +- 22 files changed, 180 insertions(+), 137 deletions(-) diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index e2e037acf..d18b7c366 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -7,7 +7,7 @@ Router has several responsibilities: Services in Kubernetes. With `--atenet-router=agentgateway`, the sidecar uses a static ConfigMap and atenet does not start an xDS server. -* ext_proc server for the proxy. To make the deployment and debugging easier, we will run this component together +* ext_proc server for the dataplane. To make the deployment and debugging easier, we will run this component together with the router, but this will be split later into its own component. * ext_proc will call into the ATE gRPC API to get the set of relevant backends (specific the worker IP) and route the traffic accordingly @@ -20,22 +20,22 @@ Router has several responsibilities: bounded wait elapses, instead of failing fast. See [docs/request-parking.md](../../../../../docs/request-parking.md). * Drains gracefully on SIGTERM: flips `/readyz` so the Service stops sending - new connections, waits out endpoint propagation (`--drain-delay`), drives - Envoy's admin API to drain established connections, gracefully stops the - ext_proc server so parked requests finish normally (`--drain-timeout`, - derived from the parking budget), then writes a drain-complete marker that - releases the Envoy container's `preStop` hook. See `drain.go` and - `envoydrain.go`. -* Authenticates actor identity on egress: on every CONNECT, the egress Envoy's - ext_proc handler re-verifies the actor's client certificate against the - actor-identity CA, reads the `ActorIdentity` X.509 extension out of it, and - checks the certified UID against the ATE API. + new connections, waits out endpoint propagation (`--drain-delay`), drains the + dataplane's established connections (Envoy only — driven over its admin API; + agentgateway manages its own termination), gracefully stops the ext_proc + server so parked requests finish normally (`--drain-timeout`, derived from + the parking budget), then writes a drain-complete marker that releases the + dataplane container's `preStop` hook. See `drain.go` and `envoydrain.go`. +* Authenticates actor identity on egress: on every CONNECT, the egress + gateway's ext_proc handler re-verifies the actor's client certificate against + the actor-identity CA, reads the `ActorIdentity` X.509 extension out of it, + and checks the certified UID against the ATE API. ## packages The ext_proc server handles both traffic directions, and they apply opposite -trust models — egress derives the actor identity from a client certificate -Envoy verified against the actor-identity CA, ingress treats every request +trust models — egress derives the actor identity from a client certificate the +gateway verified against the actor-identity CA, ingress treats every request header as unauthenticated client input — so the two are kept in separate packages that cannot reach into each other: @@ -48,8 +48,9 @@ packages that cannot reach into each other: * `egress` — certificate-based actor-identity authentication for outbound CONNECTs. -Direction is decided by the Envoy filter chain that accepted the request -(`xds.filter_chain_name`), never by anything in the request itself, so a client +Direction is decided by the filter chain the dataplane says accepted the +request (`xds.filter_chain_name`, an Envoy attribute the egress gateway is +configured to send), never by anything in the request itself, so a client cannot pick the egress path by crafting one. `router` itself does the wiring. ## modes @@ -67,9 +68,13 @@ than falling back to the other handler, which would run the request through the wrong trust model. Ingress and egress are deployed separately today — `atenet-router` fronts the -ingress Envoy, `atenet-egress` the egress one — because the two scale +ingress dataplane, `atenet-egress` the egress gateway — because the two scale independently, not because they need separate binaries. +The `--atenet-router` choice only applies to the ingress dataplane. The egress +gateway is its own Deployment with a statically configured Envoy, so +`--atenet-router=agentgateway` leaves it untouched. + ## status page Serve a `/statusz` page on port 8080. diff --git a/cmd/atenet/internal/router/cmd.go b/cmd/atenet/internal/router/cmd.go index 8d12cbbf7..018fb18d0 100644 --- a/cmd/atenet/internal/router/cmd.go +++ b/cmd/atenet/internal/router/cmd.go @@ -29,7 +29,7 @@ func NewRouterCmd() *cobra.Command { cmd := &cobra.Command{ Use: "router", - Short: "Router components including xDS server and Envoy ExtProc gateway processing server", + Short: "Router components including the Envoy xDS server and the ext_proc gateway processing server", RunE: func(cmd *cobra.Command, args []string) error { srv, err := NewRouterServer(cfg) if err != nil { @@ -41,7 +41,7 @@ func NewRouterCmd() *cobra.Command { }, } - cmd.Flags().StringVar((*string)(&cfg.Mode), "mode", string(ModeAll), fmt.Sprintf("Traffic direction this instance serves: %q (also runs the xDS server and ActorTemplate controller for the ingress Envoy), %q (ext_proc only, needs no Kubernetes access), or %q for both. The ext_proc mux refuses a direction this instance was not started to serve rather than falling back to the other one", ModeIngress, ModeEgress, ModeAll)) + cmd.Flags().StringVar((*string)(&cfg.Mode), "mode", string(ModeAll), fmt.Sprintf("Traffic direction this instance serves: %q (also runs the ingress control plane — the xDS server and ActorTemplate controller — for an Envoy dataplane), %q (ext_proc only, needs no Kubernetes access), or %q for both. The ext_proc mux refuses a direction this instance was not started to serve rather than falling back to the other one", ModeIngress, ModeEgress, ModeAll)) cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error") cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") cmd.Flags().BoolVar(&cfg.Standalone, "standalone", false, "Run in standalone mode, bypassing creation of managed deployment and services in Kubernetes cluster") @@ -51,13 +51,13 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "k8s:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.") cmd.Flags().IntVar(&cfg.HttpPort, "port-http", 8080, "TCP port for workload traffic entering through the Envoy Router") cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections") - cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server") - cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the Envoy External Processing (ext_proc) server") + cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the External Processing (ext_proc) server the dataplane calls") + cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the External Processing (ext_proc) server") cmd.Flags().StringVar(&cfg.EnvoyImage, "envoy-image", "envoyproxy/envoy:v1.30-latest", "Image URI used for dynamically launched router instances") cmd.Flags().StringVar(&cfg.TemplatesFile, "actor-templates-file", "", "Path to offline YAML configuration file listing ActorTemplates") cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)") cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services") - cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router") + cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the router dataplane") cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file.") cmd.Flags().StringVar(&cfg.UpstreamCredentialBundlePath, "upstream-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle (cert+key) the router presents as the client cert when dialing the actor's atunnel ingress server over mTLS. Empty disables upstream mTLS (legacy plaintext pod-IP:80).") cmd.Flags().StringVar(&cfg.UpstreamTrustBundlePath, "upstream-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "PEM trust bundle used to validate the actor's atunnel ingress server certificate.") @@ -86,7 +86,7 @@ func NewRouterCmd() *cobra.Command { cmd.Flags().DurationVar(&cfg.DrainDelay, "drain-delay", 13*time.Second, "How long to keep serving after SIGTERM before starting the drain, covering readiness-probe detection and Service endpoint propagation") cmd.Flags().DurationVar(&cfg.DrainTimeout, "drain-timeout", 0, "Deadline for the ext_proc drain on shutdown; streams still open past it (parked requests included) are forcefully cancelled. 0 (the default) derives --parked-request-budget + the actor route timeout + margin so parked requests always finish normally. Explicit values must be >= --parked-request-budget") cmd.Flags().StringVar(&cfg.EnvoyAdminAddr, "envoy-admin-address", "127.0.0.1:9901", "Envoy admin interface the shutdown sequence drives to drain the sidecar (healthcheck/fail, drain_listeners, stats polling). Ignored with --atenet-router=agentgateway") - cmd.Flags().StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the Envoy container's preStop hook polls for it so Envoy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake") + cmd.Flags().StringVar(&cfg.DrainCompleteFile, "drain-complete-file", defaultDrainCompleteFile, "Marker file created (on a pod-shared emptyDir) once the shutdown drain completes; the dataplane container's preStop hook polls for it so the proxy exits as soon as — and no sooner than — the drain is done. Removed at startup to defuse stale markers. Empty disables the handshake") return cmd } diff --git a/cmd/atenet/internal/router/config.go b/cmd/atenet/internal/router/config.go index 5004055f8..40876ab39 100644 --- a/cmd/atenet/internal/router/config.go +++ b/cmd/atenet/internal/router/config.go @@ -30,9 +30,9 @@ const ( // Mode selects which ext_proc directions an atenet instance serves. One binary // implements both, as two handlers behind the same ext_proc mux, but a -// deployment usually fronts one Envoy and only needs the matching direction: -// ingress and egress scale independently, so they run as separate Deployments -// (atenet-router and atenet-egress). +// deployment usually fronts one dataplane proxy and only needs the matching +// direction: ingress and egress scale independently, so they run as separate +// Deployments (atenet-router and atenet-egress). // // The mode is a floor on what an instance will answer, not just a hint. The mux // refuses a direction it has no handler for rather than falling back to the @@ -42,12 +42,15 @@ const ( type Mode string const ( - // ModeIngress serves actor-addressed traffic entering the mesh, and runs - // the xDS server and ActorTemplate controller that configure its Envoy. + // ModeIngress serves actor-addressed traffic arriving at the ingress + // gateway, and runs the ingress control plane — the xDS server and + // ActorTemplate controller — that configures its dataplane. Only the Envoy + // dataplane takes configuration from it; agentgateway is statically + // configured. ModeIngress Mode = "ingress" - // ModeEgress serves actor CONNECTs leaving the mesh. Nothing else runs: the - // egress Envoy is statically configured, so there is no xDS server, no - // ActorTemplate controller, and no Kubernetes client. + // ModeEgress serves actor CONNECTs leaving through the egress gateway. + // Nothing else runs: the egress gateway is statically configured, so there + // is no xDS server, no ActorTemplate controller, and no Kubernetes client. ModeEgress Mode = "egress" // ModeAll serves both directions from one instance. This is the default, // and what a single-gateway or local development setup wants. @@ -56,7 +59,7 @@ const ( // ServesIngress reports whether this mode answers ingress requests. It also // gates the ingress control plane: the xDS server that configures the ingress -// Envoy and the ActorTemplate controller that feeds it. +// dataplane (Envoy only) and the ActorTemplate controller that feeds it. func (m Mode) ServesIngress() bool { return m != ModeEgress } // ServesEgress reports whether this mode answers egress CONNECTs. @@ -159,8 +162,8 @@ type routerConfig struct { EnvoyAdminAddr string // DrainCompleteFile is the marker file written once shutdown completes, - // releasing Envoy's preStop hook on the shared emptyDir. Removed at startup to - // defuse stale markers. Empty disables the handshake. + // releasing the dataplane container's preStop hook on the shared emptyDir. + // Removed at startup to defuse stale markers. Empty disables the handshake. DrainCompleteFile string } diff --git a/cmd/atenet/internal/router/config_test.go b/cmd/atenet/internal/router/config_test.go index 0c4b60ec2..4e41060dd 100644 --- a/cmd/atenet/internal/router/config_test.go +++ b/cmd/atenet/internal/router/config_test.go @@ -237,7 +237,7 @@ func TestSetOtlpCollector(t *testing.T) { // No collector address may keep the router from starting. The address // defaults to OTEL_EXPORTER_OTLP_ENDPOINT, which also feeds the router's // own exporter and where https is perfectly valid; the router is the xDS - // control plane for every Envoy in the mesh, so dropping Envoy's spans is + // control plane for every ingress Envoy, so dropping Envoy's spans is // always the cheaper failure. setOtlpCollector returns nothing precisely so // this cannot regress into a startup error. tests := []struct { diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 5e2a105fe..0897e5455 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package egress implements the ext_proc handler for traffic leaving the mesh: +// Package egress implements the ext_proc handler for outbound actor traffic: // it authenticates the actor behind an egress CONNECT before the gateway // tunnels it out. // @@ -88,18 +88,9 @@ func (h *Handler) Direction() extproc.Direction { return extproc.DirectionEgress // request metadata — contributes to the identity; the only inputs are the // certificate the actor-identity CA signed and the control plane's own view of // that actor. -// -// Authorization by destination and credential/token injection are deliberately -// a TODO once we have SessionIdentity RPC service figured out. -// -// Egress never resumes an actor — it requires one already RUNNING — and picks -// no upstream of its own, so the Result carries neither a resume outcome nor a -// target. func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestMetadata) (extproc.Result, error) { - // Dispatch is by filter chain, so reaching here means the egress listener - // accepted the request. That listener only routes CONNECT (its sole route - // is a connect_matcher), so anything else is a config drift rather than a - // client the gateway should tunnel for. + // Sanity check that we were called on the Egress listener filter chain with + // a CONNECT. if !strings.EqualFold(md.Method, "CONNECT") { return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_MethodNotAllowed, "egress denied: expected CONNECT, got %q", md.Method) @@ -123,32 +114,62 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM "egress denied: invalid actor certificate") } - atespace := identity.Atespace - actorName := identity.ActorName - actorUID := identity.ActorUid - // For a CONNECT the :authority is the actor's original destination (IP:port). - destination := md.Host + if err := validateIdentity(identity); err != nil { + return extproc.Result{}, err + } + if err := h.validateActor(ctx, identity); err != nil { + return extproc.Result{}, err + } + slog.InfoContext(ctx, "egress identity authenticated", + slog.String("atespace", identity.Atespace), + slog.String("actor", identity.ActorName), + slog.String("actorUid", identity.ActorUid), + // For a CONNECT the :authority is the actor's original destination + // (IP:port). + slog.String("destination", md.Host)) + + // Identity is authenticated; let the CONNECT proceed unchanged. + return extproc.Result{ + Response: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, nil +} + +// validateIdentity checks that the identity a verified actor certificate +// carries names an actor that could exist at all, before it is used as a +// control-plane lookup key. +func validateIdentity(identity *substratex509.ActorIdentity) error { // The CA only ever mints these from control-plane state, so a name that is // not a legal resource name means the CA or its inputs are compromised. - if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) { - return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, - "egress denied: invalid actor identity %q/%q", atespace, actorName) + if !resources.IsValidResourceName(identity.Atespace) || !resources.IsValidResourceName(identity.ActorName) { + return extproc.NewReqError(envoy_type.StatusCode_Forbidden, + "egress denied: invalid actor identity %q/%q", identity.Atespace, identity.ActorName) } + return nil +} + +// validateActor checks the identity a certificate certifies against the control +// plane's current view of that actor: it still exists, it is the actor the +// certificate was issued to, and it is running. Every error it returns is +// already a client-facing ext_proc denial. +func (h *Handler) validateActor(ctx context.Context, identity *substratex509.ActorIdentity) error { + atespace := identity.Atespace + actorName := identity.ActorName + actorUID := identity.ActorUid // Confirm the certified actor still exists. The name is only a lookup key // here; the UID below is what actually authorizes. + // TODO: this can cause heavy load on ate api server. Change it based on https://github.com/agent-substrate/substrate/issues/592. actor, err := h.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: actorName}, }) if err != nil { - return extproc.Result{}, mapEgressIdentityError(atespace, actorName, err) + return mapEgressIdentityError(atespace, actorName, err) } - // Authorize on the UID, not the name. Names are reused: delete an actor and - // recreate it under the same atespace/name and it is a different actor with - // a different UID. A certificate outliving its actor must not carry over to - // the successor, so the UID the CA certified has to match the UID the + // Authorize on the UID, not the name. The UID the CA certified has to match the UID the // control plane holds right now. if uid := actor.GetMetadata().GetUid(); uid != actorUID { slog.WarnContext(ctx, "egress denied: actor UID mismatch", @@ -156,31 +177,16 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM slog.String("actor", actorName), slog.String("certificateActorUid", actorUID), slog.String("currentActorUid", uid)) - return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, + return extproc.NewReqError(envoy_type.StatusCode_Forbidden, "egress denied: actor %q/%q is not the actor this certificate was issued to", atespace, actorName) } // The actor performing egress must actually be running. if actor.GetStatus() != ateapipb.Actor_STATUS_RUNNING { - return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, + return extproc.NewReqError(envoy_type.StatusCode_Forbidden, "egress denied: actor %q/%q is %s, not running", atespace, actorName, actor.GetStatus()) } - - slog.InfoContext(ctx, "egress identity authenticated", - slog.String("atespace", atespace), - slog.String("actor", actorName), - slog.String("actorUid", actorUID), - slog.String("destination", destination), - slog.String("status", actor.GetStatus().String())) - - // Identity is authenticated; let the CONNECT proceed unchanged. Milestone 2 - // would additionally authorize `destination` and inject upstream credentials - // here by returning a HeaderMutation. - return extproc.Result{ - Response: &extprocv3.HeadersResponse{ - Response: &extprocv3.CommonResponse{}, - }, - }, nil + return nil } // authenticateActorCertificate turns the mTLS peer certificate Envoy recorded diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go index 38fe10aa7..52864d9cd 100644 --- a/cmd/atenet/internal/router/extproc/dispatch.go +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -18,15 +18,15 @@ import ( extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" ) -// Direction is the side of the mesh a request arrived on. It selects the -// handler, so it must come from something Envoy asserts rather than from the +// Direction is the gateway a request arrived through. It selects the handler, +// so it must come from something the dataplane asserts rather than from the // request. type Direction string const ( - // DirectionIngress is traffic entering the mesh, addressed to an actor. + // DirectionIngress is inbound traffic addressed to an actor. DirectionIngress Direction = "ingress" - // DirectionEgress is traffic leaving the mesh, tunneled out of an actor. + // DirectionEgress is outbound traffic tunneled out of an actor. DirectionEgress Direction = "egress" ) diff --git a/cmd/atenet/internal/router/extproc/errors.go b/cmd/atenet/internal/router/extproc/errors.go index 12b1f0672..0a6acb19a 100644 --- a/cmd/atenet/internal/router/extproc/errors.go +++ b/cmd/atenet/internal/router/extproc/errors.go @@ -54,8 +54,8 @@ func WrapReqError(code envoy_type.StatusCode, cause error, format string, args . } } -// ImmediateResponse tells Envoy to answer the request itself, without going -// upstream. +// ImmediateResponse tells the dataplane to answer the request itself, without +// going upstream. func ImmediateResponse(statusCode envoy_type.StatusCode, message string) *extprocv3.ProcessingResponse { return &extprocv3.ProcessingResponse{ Response: &extprocv3.ProcessingResponse_ImmediateResponse{ diff --git a/cmd/atenet/internal/router/extproc/extproc.go b/cmd/atenet/internal/router/extproc/extproc.go index f66fc6fd1..74153f325 100644 --- a/cmd/atenet/internal/router/extproc/extproc.go +++ b/cmd/atenet/internal/router/extproc/extproc.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package extproc implements the Envoy external processing (ext_proc) gRPC -// server that the atenet router serves to its Envoy gateways. +// Package extproc implements the external processing (ext_proc) gRPC server +// that the atenet router serves to its dataplane gateways. // // This package is the multiplexer and nothing else: it terminates the // ext_proc stream, works out which direction — ingress or egress — a request @@ -42,8 +42,8 @@ import ( // direction absent from the map is refused at the mux — see Server.Process. type Handlers map[Direction]Handler -// Server implements the Envoy external processing gRPC server, dispatching -// each request to the Handler for the direction it arrived on. +// Server implements the external processing gRPC server, dispatching each +// request to the Handler for the direction it arrived on. type Server struct { port int handlers Handlers @@ -132,12 +132,13 @@ func (s *Server) processRequestHeaders( // One atenet binary serves both directions, as two ext_proc handlers // selected here. They are deployed separately today — atenet-router fronts - // the ingress Envoy, atenet-egress the egress one — because the two scale - // independently, and --mode restricts an instance to the direction its - // deployment fronts. Nothing stops a single instance from serving both. + // the ingress dataplane, atenet-egress the egress gateway — because the two + // scale independently, and --mode restricts an instance to the direction + // its deployment fronts. Nothing stops a single instance from serving both. // - // Which handler runs is decided by the Envoy filter chain that accepted the - // request, never by anything in the request itself (see directionOf). + // Which handler runs is decided by the filter chain the dataplane says + // accepted the request, never by anything in the request itself (see + // directionOf). dir := directionOf(req) var res Result @@ -145,10 +146,10 @@ func (s *Server) processRequestHeaders( if handler, ok := s.handlers[dir]; ok { res, err = handler.HandleRequestHeaders(ctx, md) } else { - // The Envoy in front of this instance is sending traffic the instance - // was not started to serve. Refuse it rather than falling back to the - // other direction's handler: the two apply opposite trust models, so a - // fallback would run a request through the wrong one. + // The dataplane in front of this instance is sending traffic the + // instance was not started to serve. Refuse it rather than falling back + // to the other direction's handler: the two apply opposite trust + // models, so a fallback would run a request through the wrong one. err = NewReqError(envoy_type.StatusCode_NotFound, "this router does not serve %s traffic", dir) } diff --git a/cmd/atenet/internal/router/extproc/handler.go b/cmd/atenet/internal/router/extproc/handler.go index e0806631a..a7e994e8e 100644 --- a/cmd/atenet/internal/router/extproc/handler.go +++ b/cmd/atenet/internal/router/extproc/handler.go @@ -30,8 +30,8 @@ type Handler interface { // its dispatch table by it. Direction() Direction - // HandleRequestHeaders decides what Envoy should do with a request whose - // headers have just arrived. + // HandleRequestHeaders decides what the dataplane should do with a request + // whose headers have just arrived. // // A returned error denies the request: a *ReqError carries the status code // and client-safe body to answer with, anything else becomes a 500. The @@ -43,7 +43,7 @@ type Handler interface { // Result is what a handler tells the mux about a request it allowed. type Result struct { - // Response is the header mutation Envoy applies before the request + // Response is the header mutation the dataplane applies before the request // continues. Handlers that only authenticate return an empty CommonResponse. Response *extprocv3.HeadersResponse diff --git a/cmd/atenet/internal/router/extproc/metadata.go b/cmd/atenet/internal/router/extproc/metadata.go index 52f4fe7dd..7c69c58b3 100644 --- a/cmd/atenet/internal/router/extproc/metadata.go +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -26,8 +26,8 @@ import ( const AuthorityHeader = ":authority" // RequestMetadata is the request the mux hands to a direction handler: the -// HTTP headers Envoy sent, flattened and lowercased, with the pseudo-headers -// every handler needs pulled out. +// HTTP headers the dataplane sent, flattened and lowercased, with the +// pseudo-headers every handler needs pulled out. type RequestMetadata struct { // Headers holds every header, keyed by lowercased name. Headers map[string]string diff --git a/cmd/atenet/internal/router/extproc/metrics.go b/cmd/atenet/internal/router/extproc/metrics.go index 8ba0819e7..e19778ce3 100644 --- a/cmd/atenet/internal/router/extproc/metrics.go +++ b/cmd/atenet/internal/router/extproc/metrics.go @@ -36,7 +36,7 @@ import ( const ServiceName = "atenet-router" // atenet.router.route.duration measures the latency from when the ext_proc handler receives a request -// (Envoy -> EPP) until the target worker endpoint is resolved +// (dataplane -> EPP) until the target worker endpoint is resolved const routeDurationMetricName = "atenet.router.route.duration" // NewRouteDurationHistogram creates the atenet.router.route.duration histogram from diff --git a/cmd/atenet/internal/router/health.go b/cmd/atenet/internal/router/health.go index 99602fe64..8f911f696 100644 --- a/cmd/atenet/internal/router/health.go +++ b/cmd/atenet/internal/router/health.go @@ -158,7 +158,7 @@ func updateComponentHealth(health *ComponentHealth, healthy bool, msg string, ch func (rh *routerHealth) checkDataplane(ctx context.Context) (bool, string) { // The dataplane this polls is the *ingress* proxy sharing the router's pod. - // The egress Envoy is a separate, statically configured proxy on its own + // The egress gateway is a separate, statically configured proxy on its own // admin port; an egress-only router has none beside it, and probing this // address would report a permanently unhealthy dependency. if !rh.cfg.Mode.ServesIngress() { diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 30e3ca62e..a784696e6 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ingress implements the ext_proc handler for traffic entering the -// mesh: it resolves the actor a request is addressed to, resumes it through -// the control plane (parking the request while the worker pool is saturated), -// and points Envoy at the worker that ends up hosting it. +// Package ingress implements the ext_proc handler for traffic arriving at the +// ingress gateway: it resolves the actor a request is addressed to, resumes it +// through the control plane (parking the request while the worker pool is +// saturated), and points the dataplane at the worker that ends up hosting it. // // Everything reaching this handler is unauthenticated client input. The // opposite trust model — an actor identity carried by a CA-signed client @@ -72,10 +72,10 @@ func (h *Handler) ParkingStatus() ParkingStatus { return h.parking.status() } func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestMetadata) (extproc.Result, error) { slog.InfoContext(ctx, "Request", slog.String("host", md.Host)) - // Envoy doesn't propagate trace context into the ext_proc gRPC + // The dataplane doesn't propagate trace context into the ext_proc gRPC // stream's metadata — the per-request traceparent arrives in the // HTTP headers carried inside the ProcessingRequest payload. Extract - // from there so our span links to the Envoy ingress span. + // from there so our span links to the gateway's ingress span. ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(md.Headers)) ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") defer span.End() diff --git a/cmd/atenet/internal/router/router.go b/cmd/atenet/internal/router/router.go index f1cf98c35..5ce654b2a 100644 --- a/cmd/atenet/internal/router/router.go +++ b/cmd/atenet/internal/router/router.go @@ -133,7 +133,7 @@ func (s *RouterServer) Run(ctx context.Context) error { // shutdownCtx signals SIGTERM/SIGINT; kept separate from the work context // so in-flight ext_proc streams (parked requests, most of all) are not // cancelled the moment the signal arrives. drainOnShutdown drives the - // shutdown sequence: readiness flip → route-drain delay → Envoy drain → + // shutdown sequence: readiness flip → route-drain delay → dataplane drain → // ext_proc drain → stop the rest. shutdownCtx, stopSignals := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stopSignals() @@ -152,9 +152,9 @@ func (s *RouterServer) Run(ctx context.Context) error { } parkCfg := s.cfg.ParkedRequest.Normalized() - // The drain-complete marker persists container restarts (emptyDir); a - // stale one would release the Envoy preStop hook the moment a later drain - // begins. + // The drain-complete marker persists container restarts (emptyDir); a stale + // one would release the dataplane container's preStop hook the moment a + // later drain begins. removeStaleDrainMarker(ctx, s.cfg.DrainCompleteFile) serverboot.InitLogger() @@ -263,8 +263,8 @@ func (s *RouterServer) Run(ctx context.Context) error { s.health = newRouterHealth(s.cfg.HealthInterval, s.clientset, s.apiClient, s.cfg) - // The dataplane control plane — the xDS server and the ActorTemplate - // controller — configures the *ingress* Envoy. The egress Envoy is + // The ingress control plane — the xDS server and the ActorTemplate + // controller — configures the *ingress* dataplane. The egress gateway is // statically configured, so an egress-only instance runs neither and needs // no Kubernetes access. if s.cfg.Mode.ServesIngress() { @@ -281,7 +281,7 @@ func (s *RouterServer) Run(ctx context.Context) error { }) // Start ExtProc Server. Driven by the drain sequence rather than context - // cancel: ext_proc is failClosed, so it must outlive Envoy's drain. + // cancel: ext_proc is failClosed, so it must outlive the dataplane's drain. extprocGRPC := s.extprocSrv.NewGRPCServer() g.Go(func() error { slog.InfoContext(ctx, "Starting ExtProc Server", slog.Int("port", s.cfg.ExtprocPort)) @@ -356,8 +356,8 @@ func (s *RouterServer) Run(ctx context.Context) error { // OTEL_EXPORTER_OTLP_ENDPOINT, which the router's own exporter reads too and // which legitimately carries forms Envoy's plaintext tracer cluster cannot // reach — an https collector, most of all. Refusing to start would take the -// xDS control plane for every Envoy in the mesh down over a tracing endpoint -// that works fine for its other reader. Losing Envoy's spans is the smaller +// xDS control plane for every ingress Envoy down over a tracing endpoint that +// works fine for its other reader. Losing Envoy's spans is the smaller // failure, so take it and say so loudly. func setOtlpCollector(ctx context.Context, xdsSrv *XdsServer, addr string) { if err := xdsSrv.SetOtlpCollector(addr); err != nil { diff --git a/demos/egress/README.md b/demos/egress/README.md index 763aac7e7..6e5dcf85a 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -47,7 +47,7 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r pod as an ext_proc sidecar, the same binary that serves ingress, started with `--mode=egress`) re-verifies the chain, requires exactly one `ActorIdentity` extension with `purpose: atunnel`, and calls the ate API (`GetActor`). It returns **403** unless the certified **UID** matches a - real, `RUNNING` actor. This mirrors the ingress gateway's Envoy + ext_proc co-location; a + real, `RUNNING` actor. This mirrors the ingress gateway's dataplane + ext_proc co-location; a standalone/shared ext_proc is a future step. ## Components @@ -126,7 +126,7 @@ kubectl -n ate-system logs deploy/atenet-egress | grep '\[egress\]' # The co-located ext_proc sidecar logs the identity decision, including the UID it authorized on: kubectl -n ate-system logs deploy/atenet-egress -c ext-proc | grep -i 'egress identity\|egress denied' -# egress identity authenticated atespace=demo actor=egress-demo actorUid=… status=STATUS_RUNNING +# egress identity authenticated atespace=demo actor=egress-demo actorUid=… destination=:80 ``` The `whoami` body shows `RemoteAddr: ` — proof the request egressed diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index 63124bfb4..3c57b4d5a 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -474,7 +474,7 @@ advertises the 4318 HTTP endpoint. use.** Envoy's tracer cluster is plaintext h2c, so an `https://` endpoint is neither honored nor silently downgraded: the router logs a warning, turns Envoy-side tracing off, and starts normally. Its own spans are unaffected. -Taking the xDS control plane down for every Envoy in the mesh over a tracing +Taking the xDS control plane down for every ingress Envoy over a tracing endpoint that works fine for the router's own exporter would be the larger failure. diff --git a/docs/roadmap.md b/docs/roadmap.md index 9663abc75..3d6584509 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -95,7 +95,7 @@ Below is a collection of finer-grained efforts which we believe align with the a * Agent Development Kit (ADK) Native Support: Developing first-class bindings for ADK, allowing developers to build stateful agents that natively leverage Substrate’s lifecycle management and persistent working memory. * LangChain Remote Execution Provider: A dedicated provider for LangChain to run complex, long-running agent tools in durable, sandboxed environments. * Native MCP Server Hosting: Built-in support for deploying Model Context Protocol (MCP) servers as managed Substrate Actors, creating a secure tool ecosystem for any LLM. -* Actor-to-Actor (A2A) Calling Model: Standardized protocol for actors to discover and call other actors within the mesh via the gateway. +* Actor-to-Actor (A2A) Calling Model: Standardized protocol for actors to discover and call other actors within Substrate via the gateway. * Native MCP Tool Hosting: Ability to define and deploy standard Model Context Protocol (MCP) servers as managed Substrate Actors, providing a plug-and-play ecosystem for agentic tools. ### Operability diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh index abd641de2..19c69cfd9 100755 --- a/hack/verify-egress-demo.sh +++ b/hack/verify-egress-demo.sh @@ -19,8 +19,7 @@ # hack/install-ate-kind.sh --deploy-demo-egress # # The egress demo Actor accepts {"url":"..."} and performs an HTTP GET. With -# egress turned on (ate-api-server --egress-gateway-address, which ateapi stamps -# onto every atelet Run/Restore), the Actor's outbound TCP is nftables-REDIRECTed +# egress turned on, the Actor's outbound TCP is nftables-REDIRECTed # into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the Envoy egress # gateway, which terminates CONNECT and tunnels to the real destination. This # script drives that path and shows the gateway's access log proving the actor's diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index f168e397e..6951befd3 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -72,7 +72,7 @@ func (c *RouterClient) Close() { c.stop() } -// Get issues GET path to actor through the router, setting the actor's mesh Host +// Get issues GET path to actor through the router, setting the actor's DNS Host // so the router routes (and resumes) it. The caller must close the body. func (c *RouterClient) Get(ctx context.Context, actorRef resources.ActorRef, path string) (*http.Response, error) { return c.request(ctx, http.MethodGet, actorRef, path, nil) diff --git a/internal/resources/actorref.go b/internal/resources/actorref.go index 7d287ad8a..0beabab26 100644 --- a/internal/resources/actorref.go +++ b/internal/resources/actorref.go @@ -47,7 +47,7 @@ func (r ActorRef) LogValue() slog.Value { ) } -// DNSName returns the mesh DNS name the actor is reachable at. +// DNSName returns the uniform DNS name the actor is reachable at. // This is: "..actors.resources.substrate.ate.dev". func (r ActorRef) DNSName() string { return r.Name + "." + r.Atespace + "." + ActorDNSSuffix diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 3776a8ea1..591ef31fc 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -76,13 +76,7 @@ data: codec_type: HTTP1 upgrade_configs: - upgrade_type: CONNECT - # SEE(lior): this pair is how the actor's certificate reaches the - # ext_proc handler. ext_proc can request Envoy attributes, but none - # of them carry a custom X.509 extension, so the only way to check - # ActorIdentity in Go is to have Envoy hand over the raw chain. - # SANITIZE_SET drops whatever x-forwarded-client-cert the client - # sent and writes Envoy's own view of the verified peer, and - # chain: true puts the full URL-encoded PEM chain in it. + # TODO(liorlieberman): Can we make this cleaner? forward_client_cert_details: SANITIZE_SET set_current_client_cert_details: chain: true @@ -155,7 +149,7 @@ data: clusters: # ext_proc gRPC server = the atenet router, co-located in this pod as a # sidecar and called over localhost (same topology the ingress gateway uses - # for its Envoy + ext_proc). + # for its dataplane + ext_proc). - name: ext_proc_server type: STATIC lb_policy: ROUND_ROBIN @@ -210,6 +204,7 @@ spec: sysctls: - name: net.ipv4.ip_unprivileged_port_start value: "0" + terminationGracePeriodSeconds: 60 containers: - name: envoy image: envoyproxy/envoy:v1.34-latest @@ -227,6 +222,26 @@ spec: - atenet-egress - --service-cluster - atenet-egress + # Prevents Envoy from fast-exiting on SIGTERM before the ext-proc + # sidecar finishes its drain sequence. Polls for the drain-complete + # marker the sidecar writes on the shared emptyDir, terminating Envoy + # as soon as the drain completes (or at terminationGracePeriodSeconds + # if the sidecar crashes). + # + # TODO(liorlieberman): decide the drain policy for long-lived CONNECT + # tunnels. envoyDrainer polls downstream_cx_active until it reaches + # zero, which suits ingress (short request/response connections) but + # not egress: atunnel holds a tunnel open for the life of the actor's + # connection, so an actor still streaming when the rollout starts keeps + # the count above zero until the ~15s window expires. Every rollout + # with active egress will therefore log "N downstream connections still + # active at the drain deadline". Either accept that as the honest + # outcome (we tried to drain, then cut), or give egress its own shorter + # window so we stop pretending a tunnel will close on its own. + lifecycle: + preStop: + exec: + command: ["sh", "-c", "while [ ! -f /var/run/atenet/drain-complete ]; do sleep 0.5; done"] ports: - name: https containerPort: 443 @@ -256,6 +271,9 @@ spec: - name: actor-id-ca-certs mountPath: /run/actor-id-ca-certs readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + readOnly: true # Co-located ext_proc server: the same atenet router binary the ingress # gateway runs, started with --mode=egress so it serves only the egress # ext_proc handler (no xDS server, no ActorTemplate controller, no @@ -282,6 +300,11 @@ spec: # denies every egress CONNECT with a 503. - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt - --otlp-collector-address= + # The egress Envoy's admin listener is on 15000, not the 9901 the flag + # defaults to (that is the ingress gateway's port). Without this the + # drain sequence dials a closed port, reads the connection refusal as + # "Envoy already exited", and reports a drain it never performed. + - --envoy-admin-address=127.0.0.1:15000 env: - name: POD_NAME valueFrom: @@ -311,10 +334,14 @@ spec: - name: actor-id-ca-certs mountPath: /run/actor-id-ca-certs readOnly: true + - name: drain-signal + mountPath: /var/run/atenet volumes: - name: config configMap: name: atenet-egress + - name: drain-signal + emptyDir: {} - name: servicedns projected: sources: diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index e794e5391..1dc2e3fe1 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -161,9 +161,11 @@ spec: image: ko://github.com/agent-substrate/substrate/cmd/atenet args: - "router" - # Serve the ingress direction only: the xDS server and ActorTemplate - # controller for the Envoy in this pod, plus the ingress ext_proc - # handler. Egress runs the same binary with --mode=egress in its own + # Serve the ingress direction only: the ingress ext_proc handler, plus + # the xDS server and ActorTemplate controller that configure the + # dataplane in this pod when it is Envoy (the agentgateway overlay is + # statically configured and starts neither). + # Egress runs the same binary with --mode=egress in its own # Deployment (manifests/ate-install/atenet-egress.yaml), because the two # directions scale independently. - "--mode=ingress"