From 091968215a026a57d74f5479eec24aa17c2b1d10 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 13 Aug 2026 10:43:50 -0700 Subject: [PATCH 01/10] hack: fix DNS on IPv6-only kind clusters On a fresh IP_FAMILY=ipv6 cluster nothing resolves from inside a pod and no actor boots: CoreDNS inherits the node's IPv4 resolver, which a v6-only pod cannot reach. Two Corefile clauses fix it -- a hosts entry mapping kind-registry to its IPv6 address, and a forward to an IPv6 upstream, overridable with IPV6_DNS_UPSTREAM. IPv4 and dual-stack clusters are unchanged, and atenet-egress still crashloops on v6-only for an unrelated Envoy bind bug. (cherry picked from commit e336c22c14da25418b12cf5493d6091a6ee4450c) --- hack/create-kind-cluster.sh | 74 +++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c95..b087a5d06 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -196,6 +199,77 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry +# +# CoreDNS runs dnsPolicy: Default, inheriting the node's IPv4 resolver, which +# no pod here can reach, so external lookups SERVFAIL. Step 3's registry +# wiring is node-side, so it misses atelet too: that pull runs in atelet's own +# netns, where "kind-registry" NXDOMAINs. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + # fallthrough is load-bearing: without it every name but the registry + # NXDOMAINs, cluster.local included -- one outage traded for a worse one. + search="forward . /etc/resolv.conf" + replace="hosts { + ${reg_v6} ${reg_name} + fallthrough + } + forward . ${IPV6_DNS_UPSTREAM}" + # Unquoted on purpose: bash 3.2 splices the quotes in literally. First match + # only, and the emitted block inherits the matched line's indentation. + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Probe from a pod, not the node: the node is dual-stack and passes either + # way. The registry leg fetches rather than resolves -- the hosts entry is + # AAAA-only, which fails nslookup's A query but satisfies getaddrinfo. + echo "Verifying DNS from a pod..." + # Markers, not the exit status: --attach returns only the last leg's. + # PROBE_RAN separates a failed leg from a pod that never ran -- a different fix. + probe="$(kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "echo PROBE_RAN + nslookup storage.googleapis.com >/dev/null && echo RESOLVE_OK + wget -q -T10 -O/dev/null http://${reg_name}:5000/v2/ && echo REGISTRY_OK")" || true + if [[ "${probe}" != *PROBE_RAN* ]]; then + echo "error: the probe pod never ran, so CoreDNS is unverified" >&2 + echo " check that it scheduled and that 'busybox:1.36' pulled" >&2 + exit 1 + fi + if [[ "${probe}" != *RESOLVE_OK* ]]; then + echo "error: a pod cannot resolve an external name" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + exit 1 + fi + if [[ "${probe}" != *REGISTRY_OK* ]]; then + echo "error: DNS works but a pod cannot reach '${reg_name}' at [${reg_v6}]:5000" >&2 + echo " check the registry container is up and on the 'kind' network" >&2 + exit 1 + fi +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat < Date: Tue, 18 Aug 2026 22:04:09 -0700 Subject: [PATCH 02/10] hack: retry the IPv6-only kind DNS probe instead of asking once The probe that checks a pod can resolve an external name asked once, right after the CoreDNS rollout reported complete, and a brand-new cluster is not ready by then -- it failed the create on two of three fresh clusters here while the cluster was merely still settling. Retrying inside the pod does not rescue it, because a pod that fails keeps failing for the next 36 seconds while a fresh pod ten seconds later resolves on its first try, so the whole probe pod now runs up to four times. Five consecutive fresh clusters came up clean. (cherry picked from commit 533811a5e584d4661995c7f1a9855d60b12ffc91) --- hack/create-kind-cluster.sh | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index b087a5d06..e49796078 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -248,11 +248,32 @@ if [[ "${IP_FAMILY}" == "ipv6" ]]; then echo "Verifying DNS from a pod..." # Markers, not the exit status: --attach returns only the last leg's. # PROBE_RAN separates a failed leg from a pod that never ran -- a different fix. - probe="$(kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$" \ - --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ - sh -c "echo PROBE_RAN - nslookup storage.googleapis.com >/dev/null && echo RESOLVE_OK - wget -q -T10 -O/dev/null http://${reg_name}:5000/v2/ && echo REGISTRY_OK")" || true + # Retry the whole pod, not just the query. Asking once right after the rollout + # fails on a brand-new cluster, and retrying inside the pod does not rescue it: + # measured here, a pod that fails keeps failing for 36s while a fresh pod 10s + # later resolves first try. So the outer loop is the one that matters; the + # inner ones only absorb a slow answer. busybox nslookup has no timeout flag, + # so every budget below is attempts, not seconds. + probe="" + for probe_attempt in 1 2 3 4; do + probe="$(kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$-${probe_attempt}" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "echo PROBE_RAN + for i in 1 2 3; do + out=\$(nslookup storage.googleapis.com 2>&1) && { echo RESOLVE_OK; break; } + echo \"resolve attempt failed: \$(echo \"\$out\" | tail -2 | tr '\n' ' ')\" + sleep 3 + done + for i in 1 2 3; do + wget -q -T10 -O/dev/null http://${reg_name}:5000/v2/ && { echo REGISTRY_OK; break; } + sleep 3 + done")" || true + [[ "${probe}" == *RESOLVE_OK* && "${probe}" == *REGISTRY_OK* ]] && break + if [[ "${probe_attempt}" != 4 ]]; then + echo " the cluster is not resolving yet; re-probing (${probe_attempt}/4)..." + sleep 10 + fi + done if [[ "${probe}" != *PROBE_RAN* ]]; then echo "error: the probe pod never ran, so CoreDNS is unverified" >&2 echo " check that it scheduled and that 'busybox:1.36' pulled" >&2 @@ -261,6 +282,8 @@ if [[ "${IP_FAMILY}" == "ipv6" ]]; then if [[ "${probe}" != *RESOLVE_OK* ]]; then echo "error: a pod cannot resolve an external name" >&2 echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + echo " probe output was:" >&2 + while IFS= read -r line; do echo " ${line}" >&2; done <<<"${probe}" exit 1 fi if [[ "${probe}" != *REGISTRY_OK* ]]; then From 6d94c78b07396e0707bac665d041ce72bc4ca627 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Sat, 8 Aug 2026 10:58:33 -0700 Subject: [PATCH 03/10] atenet/router: bind the Envoy ingress listeners dual-stack The HTTP and HTTPS ingress listeners bound 0.0.0.0 only, so on a dual-stack cluster Envoy answered on the router Service's IPv4 ClusterIP and on nothing at all for IPv6. Each primary socket now carries an additional "::" address on the same port. Ipv4Compat stays false on it: setting it would clear IPV6_V6ONLY and collide with the primary already bound to that port, and Envoy rejects the whole listener when an additional address fails to bind, taking down all ingress rather than the IPv6 half. IPv4-only clusters are unaffected -- the primary is untouched, and a host without IPv6 simply has no second socket to bind. First of three commits binding atenet's gateways dual-stack. (cherry picked from commit ed822b543546d735336b3dad8187dbd77f0a3148) --- cmd/atenet/internal/router/xds.go | 23 +++++++++++++++++ cmd/atenet/internal/router/xds_test.go | 35 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e2d76d8bf..8a02f4e23 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -1108,6 +1108,27 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { } } +// dualStackAdditionalAddresses returns the IPv6 half of a dual-stack ingress +// listener, to pair with a primary 0.0.0.0 socket on the same port. Ipv4Compat +// stays false: clearing IPV6_V6ONLY would collide with that primary. +func dualStackAdditionalAddresses(port uint32) []*listenerv3.AdditionalAddress { + return []*listenerv3.AdditionalAddress{ + { + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Address: "::", + Ipv4Compat: false, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + }, + } +} + func (x *XdsServer) buildListener() *listenerv3.Listener { hcm := x.buildHcm("ingress_http", true) @@ -1123,6 +1144,7 @@ func (x *XdsServer) buildListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.ingressPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -1182,6 +1204,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.httpsPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 6fa5c428b..8a7046ec2 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -158,6 +158,22 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if sa.GetAddress() != "0.0.0.0" { t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8081 { + t.Errorf("Expected additional port 8081, got %d", asa.GetPortValue()) + } } } @@ -196,6 +212,25 @@ func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { if sa.GetPortValue() != 8443 { t.Errorf("Expected port 8443, got %d", sa.GetPortValue()) } + if sa.GetAddress() != "0.0.0.0" { + t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) + } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPSListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8443 { + t.Errorf("Expected additional port 8443, got %d", asa.GetPortValue()) + } // Verify the TLS config references the serving cert via SDS rather // than embedding it: inline filename DataSources are read only once From 301bf6f706e5c3297c59afc2fae74ed46fc59873 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 12 Aug 2026 21:41:31 -0700 Subject: [PATCH 04/10] atenet/router: bind the admin socket and Service dual-stack The Envoy admin socket bound 0.0.0.0, and the atenet-router Service carried no ipFamilyPolicy -- which the API server defaults to SingleStack, one IPv4 ClusterIP and nothing else. Between them the router had no IPv6 address to answer on. The socket now binds "::" with ipv4_compat, one socket for both families, and the Service asks for PreferDualStack. ipv4_compat is load-bearing rather than incidental: dataplane.go health-checks the admin listener over http://127.0.0.1:9901/ready, so a bare "::" would report the dataplane component of /statusz unhealthy. Prefer, not Require, keeps the Service valid on a single-stack cluster, where it is a no-op; spec.ipFamilies is left alone because the primary family is immutable and the API server appends the secondary itself. (cherry picked from commit d5edac3d3cf5568ed0ca17a061331bacf4594db1) --- manifests/ate-install/atenet-router.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index e05e06efb..82d729ee5 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -86,7 +86,10 @@ data: admin: address: socket_address: - address: 0.0.0.0 + # ipv4_compat clears IPV6_V6ONLY, so this one socket serves both + # families; dataplane.go probes /ready over the IPv4 loopback. + address: "::" + ipv4_compat: true port_value: 9901 node: @@ -354,6 +357,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-router ports: From f804208a38f65c76b2a8868f6f13915e9738883d Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 13 Aug 2026 07:20:18 -0700 Subject: [PATCH 05/10] atenet/egress: bind the Envoy sockets and Service dual-stack The gateway's admin and :443 sockets bound 0.0.0.0, so on an IPv6-primary cluster the kubelet probed the pod on its only address and atenet-egress crashlooped -- Envoy started fine and logged "admin address: 0.0.0.0:15000" -- while an actor's CONNECT had no v6 path in. Both sockets now bind "::" with ipv4_compat, and the Service asks for PreferDualStack so a dual-stack cluster hands out an IPv6 ClusterIP to reach them on. ipv4_compat matters on the admin socket in particular: the ext-proc sidecar's drainer dials 127.0.0.1:15000, and envoydrain.go reads a refusal there as "Envoy already exited" and skips the drain silently. (cherry picked from commit d480c6fc8a9040472ba52528da94bcdb04f739d8) --- manifests/ate-install/atenet-egress.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fc..808a828e7 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -37,12 +37,13 @@ data: envoy.yaml: | admin: address: - socket_address: { address: 0.0.0.0, port_value: 15000 } + # ipv4_compat: the drainer dials this on IPv4 loopback (--envoy-admin-address, below). + socket_address: { address: "::", ipv4_compat: true, port_value: 15000 } static_resources: listeners: - name: egress address: - socket_address: { address: 0.0.0.0, port_value: 443 } + socket_address: { address: "::", ipv4_compat: true, port_value: 443 } filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in @@ -379,6 +380,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-egress ports: From c91689fa17d2c247135fe12f25f3d7e573c95491 Mon Sep 17 00:00:00 2001 From: Suraj Kumar Date: Sat, 15 Aug 2026 12:57:09 +0000 Subject: [PATCH 06/10] ateomnet: enable IPv6 forwarding in worker pod netns EnableIPv4Forwarding now also writes /proc/sys/net/ipv6/conf/all/forwarding so actor IPv6 traffic (including DNS queries) is routed between the actor veth and pod eth0 instead of being dropped by ip6_forward() on dual-stack / IPv6-only clusters. Factor the sysctl write into writeSysctlIfUnset preserving the original read-only remount/restore behavior, and add unit coverage for its fast paths. Fixes: agent-substrate/substrate#945 (cherry picked from commit 14e98ae1ff4c29c514e924f0a76a2f27f8e9d11b) --- internal/ateomnet/net.go | 28 ++++++++- internal/ateomnet/write_sysctl_test.go | 84 ++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 internal/ateomnet/write_sysctl_test.go diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 91203a8e0..2d3daa473 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -193,6 +193,9 @@ func PodIPv4() (net.IP, error) { } // EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. +// It also enables IPv6 forwarding so actor IPv6 traffic (including DNS queries +// on IPv6-capable clusters) is routed between the veth and eth0 instead of +// being dropped by ip6_forward(). func EnableIPv4Forwarding() error { // Forwarding is required because actor packets now enter the worker pod via // the host-side veth and then leave through the pod's eth0. Without this, the @@ -203,20 +206,41 @@ func EnableIPv4Forwarding() error { // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag // is not locked: clear it, write the sysctl, restore ro. const path = "/proc/sys/net/ipv4/ip_forward" + if err := writeSysctlIfUnset(path); err != nil { + return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) + } + // IPv6 forwarding: actor packets that arrive on the veth and leave via eth0 + // are IPv6 on dual-stack / IPv6-only clusters. Without + // net.ipv6.conf.all.forwarding the kernel drops every IPv6 packet in + // ip6_forward(), including the actor's DNS queries. conf.all.forwarding=1 + // also implies the per-interface default, so a single write covers the veth + // and eth0. + const v6path = "/proc/sys/net/ipv6/conf/all/forwarding" + if err := writeSysctlIfUnset(v6path); err != nil { + return fmt.Errorf("while enabling IPv6 forwarding in worker pod netns: %w", err) + } + return nil +} + +// writeSysctlIfUnset writes "1\n" to a sysctl path unless it already reads "1". +func writeSysctlIfUnset(path string) error { if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { return nil } if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { return nil } + // Without privileged, the container runtime bind-mounts /proc/sys read-only. + // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag + // is not locked: clear it, write the sysctl, restore ro. if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { - return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err) + return fmt.Errorf("while remounting /proc/sys read-write to enable forwarding: %w", err) } defer func() { _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") }() if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { - return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) + return fmt.Errorf("while writing %s: %w", path, err) } return nil } diff --git a/internal/ateomnet/write_sysctl_test.go b/internal/ateomnet/write_sysctl_test.go new file mode 100644 index 000000000..b4cb1db16 --- /dev/null +++ b/internal/ateomnet/write_sysctl_test.go @@ -0,0 +1,84 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "os" + "path/filepath" + "testing" +) + +// TestWriteSysctlIfUnset verifies writeSysctlIfUnset's fast paths against a +// temp file standing in for a /proc/sys node: it must not rewrite a value +// that already reads "1", and it must write "1\n" when the value is missing +// or unset. The privileged bind-remount path is covered by the netns +// integration tests (withTestNetNS), which require root. +func TestWriteSysctlIfUnset(t *testing.T) { + dir := t.TempDir() + + t.Run("already_set", func(t *testing.T) { + p := filepath.Join(dir, "already") + // Sentinel content: if writeSysctlIfUnset rewrote the file, the value + // would change to "1\n" and this assertion would fail. Keeping the + // file larger than the helper's output makes a silent rewrite + // detectable. + if err := os.WriteFile(p, []byte("1 other-content\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if string(b) != "1 other-content\n" { + t.Fatalf("already-set file was rewritten: %q", b) + } + }) + + t.Run("unset_written", func(t *testing.T) { + p := filepath.Join(dir, "unset") + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if len(b) < 1 || b[0] != '1' { + t.Fatalf("expected '1' written, got %q", b) + } + }) + + t.Run("zero_is_rewritten", func(t *testing.T) { + p := filepath.Join(dir, "zero") + if err := os.WriteFile(p, []byte("0\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if len(b) < 1 || b[0] != '1' { + t.Fatalf("expected '1' written, got %q", b) + } + }) +} From 3055db5e5e1338baac25f802a99998a86209f3b6 Mon Sep 17 00:00:00 2001 From: Suraj Kumar Date: Sun, 16 Aug 2026 19:12:06 +0000 Subject: [PATCH 07/10] ateomnet: return nil when sysctl path missing in writeSysctlIfUnset IPv6 sysctls are absent on kernels with IPv6 disabled (e.g. some containers set net.ipv6.conf.* only when IPv6 is enabled). Treat a missing path as 'nothing to enable' instead of forcing a remount and failing, matching the documented behavior. (cherry picked from commit b01a8b4b3c62614841e38fa14dfa8b0ddeb68731) --- internal/ateomnet/net.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 2d3daa473..a2a950178 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -223,6 +223,8 @@ func EnableIPv4Forwarding() error { } // writeSysctlIfUnset writes "1\n" to a sysctl path unless it already reads "1". +// If the path does not exist (e.g. IPv6 sysctls on a kernel with IPv6 disabled), +// it returns nil — IPv6 forwarding is simply unavailable, not an error. func writeSysctlIfUnset(path string) error { if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { return nil @@ -230,6 +232,10 @@ func writeSysctlIfUnset(path string) error { if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { return nil } + if _, err := os.Stat(path); os.IsNotExist(err) { + // Path absent (e.g. IPv6 disabled in kernel): nothing to enable. + return nil + } // Without privileged, the container runtime bind-mounts /proc/sys read-only. // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag // is not locked: clear it, write the sysctl, restore ro. From 1ea34bc4767b0b30444f1234668e3d17b5aa6251 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Mon, 10 Aug 2026 20:57:30 +0000 Subject: [PATCH 08/10] ateomnet: give the actor veth an IPv6 address and dual-stack nftables rules Add an fd00:169:254::/126 point-to-point pair alongside the existing IPv4 addresses, with a matching ::/0 default route, and move the actor nftables table from ip to inet so a single table carries both families. Each payload match now guards on NFPROTO to stay off the other family's packets, and teardown lists the inet family too: naming the wrong family there dumps empty, takes the "already clean" path, and silently leaks the table. Assign the IPv6 addresses with IFA_F_NODAD instead of writing the accept_dad sysctl. The ateom container is unprivileged, so containerd mounts /proc/sys read-only and the write failed with EROFS, taking SetupActorNetwork and every actor start down with it on both sandbox classes. A root-gated assertion pins the flag; the existing tests run as real root, where the sysctl is writable and the bug is invisible. (cherry picked from commit 96374378f6730e050e270dca74952a4c0816f89d) --- internal/ateomnet/net.go | 109 ++++++++++++++++++++++++++-- internal/ateomnet/net_linux_test.go | 63 ++++++++++++++++ 2 files changed, 167 insertions(+), 5 deletions(-) diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index a2a950178..2e1d27dfa 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -43,6 +43,11 @@ const ( ActorVethIP = "169.254.17.2" ActorNftTableName = "ateom_actor" + HostVethIPv6CIDR = "fd00:169:254::1/126" + ActorVethIPv6CIDR = "fd00:169:254::2/126" + ActorVethIPv6Gateway = "fd00:169:254::1" + ActorVethIPv6IP = "fd00:169:254::2" + // ActorVethSubnet is the point-to-point /30 the actor veth lives on. ActorVethSubnet = "169.254.17.0/30" ) @@ -51,6 +56,10 @@ var ( HostVethAddr = MustParseAddr(HostVethCIDR) ActorVethAddr = MustParseAddr(ActorVethCIDR) ActorVethGwIP = MustParseIP(ActorVethGateway) + + HostVethIPv6Addr = mustParseNoDADAddr(HostVethIPv6CIDR) + ActorVethIPv6Addr = mustParseNoDADAddr(ActorVethIPv6CIDR) + ActorVethIPv6GwIP = MustParseIPv6(ActorVethIPv6Gateway) ) // MustParseAddr parses a CIDR string into a netlink.Addr, panicking on error. @@ -62,6 +71,18 @@ func MustParseAddr(cidr string) *netlink.Addr { return a } +// mustParseNoDADAddr parses a CIDR into an address flagged IFA_F_NODAD. +// +// Per-address flag rather than the interface-wide accept_dad sysctl because the +// ateom container is unprivileged, so containerd mounts /proc/sys read-only. +// DAD is pointless on a point-to-point veth nobody else can reach, and it would +// otherwise hold the address tentative for ~1s on every resume. +func mustParseNoDADAddr(cidr string) *netlink.Addr { + a := MustParseAddr(cidr) + a.Flags = unix.IFA_F_NODAD + return a +} + // MustParseIP parses an IPv4 string into a net.IP, panicking on error. func MustParseIP(s string) net.IP { ip := net.ParseIP(s).To4() @@ -71,6 +92,15 @@ func MustParseIP(s string) net.IP { return ip } +// MustParseIPv6 parses an IPv6 string into a net.IP, panicking on error. +func MustParseIPv6(s string) net.IP { + ip := net.ParseIP(s).To16() + if ip == nil { + panic(fmt.Sprintf("parsing constant IPv6 %q", s)) + } + return ip +} + // MustParseMAC parses a MAC address string into a net.HardwareAddr, panicking on error. func MustParseMAC(s string) net.HardwareAddr { m, err := net.ParseMAC(s) @@ -107,6 +137,10 @@ func ConfigureActorVeth(ctx context.Context) error { if err := netlink.AddrReplace(actorLink, ActorVethAddr); err != nil { return fmt.Errorf("while assigning actor veth address: %w", err) } + if err := netlink.AddrReplace(actorLink, ActorVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning actor veth ipv6 address: %w", err) + } + if err := netlink.LinkSetUp(actorLink); err != nil { return fmt.Errorf("while bringing up actor veth: %w", err) } @@ -117,6 +151,13 @@ func ConfigureActorVeth(ctx context.Context) error { }); err != nil { return fmt.Errorf("while installing actor default route: %w", err) } + if err := netlink.RouteReplace(&netlink.Route{ + LinkIndex: actorLink.Attrs().Index, + Gw: ActorVethIPv6GwIP, + Dst: &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}, + }); err != nil { + return fmt.Errorf("while installing actor default ipv6 route: %w", err) + } return nil } @@ -259,9 +300,6 @@ func InstallActorNftablesRules(egressPort uint16) error { // rules in an ateom-owned table makes cleanup simple and avoids mutating // Kubernetes or CNI-managed chains directly. // - // TODO: Add IPv6 veth addressing, forwarding, and nftables rules once actor - // networking supports dual-stack pods. The current actor network is IPv4-only. - // // The rules do three things: // // * prerouting: redirect new actor TCP connections to atunnel's local @@ -278,7 +316,7 @@ func InstallActorNftablesRules(egressPort uint16) error { c := &nftables.Conn{} table := &nftables.Table{ - Family: nftables.TableFamilyIPv4, + Family: nftables.TableFamilyINet, Name: ActorNftTableName, } c.AddTable(table) @@ -293,6 +331,9 @@ func InstallActorNftablesRules(egressPort uint16) error { if redirectRule := ActorEgressRedirectRule(table, prerouting, egressPort); redirectRule != nil { c.AddRule(redirectRule) } + if redirectRuleIPv6 := ActorIPv6EgressRedirectRule(table, prerouting, egressPort); redirectRuleIPv6 != nil { + c.AddRule(redirectRuleIPv6) + } postrouting := c.AddChain(&nftables.Chain{ Name: "postrouting", @@ -306,6 +347,11 @@ func InstallActorNftablesRules(egressPort uint16) error { Chain: postrouting, Exprs: append(IPSourceEqual(ActorVethIP), &expr.Masq{}), }) + c.AddRule(&nftables.Rule{ + Table: table, + Chain: postrouting, + Exprs: append(IPv6SourceEqual(ActorVethIPv6IP), &expr.Masq{}), + }) acceptPolicy := nftables.ChainPolicyAccept forward := c.AddChain(&nftables.Chain{ @@ -336,7 +382,7 @@ func RemoveActorNftablesRules() error { // per-worker and currently per-active-actor because this worker path runs at // most one actor at a time. Missing tables are treated as already clean. c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) + tables, err := c.ListTablesOfFamily(nftables.TableFamilyINet) if err != nil { return fmt.Errorf("while listing nftables tables: %w", err) } @@ -359,6 +405,12 @@ func IPSourceEqual(ip string) []expr.Any { func IPPayloadEqual(offset uint32, ip string) []expr.Any { return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.NFPROTO_IPV4}, + }, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, @@ -373,6 +425,32 @@ func IPPayloadEqual(offset uint32, ip string) []expr.Any { } } +func IPv6SourceEqual(ip string) []expr.Any { + return IPv6PayloadEqual(8, ip) +} + +func IPv6PayloadEqual(offset uint32, ip string) []expr.Any { + return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.NFPROTO_IPV6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: 16, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: MustParseIPv6(ip), + }, + } +} + func TCPProtocol() []expr.Any { return []expr.Any{ &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, @@ -402,6 +480,24 @@ func ActorEgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} } +// ActorIPv6EgressRedirectRule is ActorEgressRedirectRule for the actor's IPv6 +// source address. Both rules live in the same inet table, so each carries its +// own NFPROTO match to keep it off the other family's packets. +func ActorIPv6EgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port uint16) *nftables.Rule { + if port == 0 { + return nil + } + exprs := append(IPv6SourceEqual(ActorVethIPv6IP), TCPProtocol()...) + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(port), + }, + &expr.Redir{RegisterProtoMin: 1}, + ) + return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} +} + // CreateNetNSWithoutSwitching creates a named netns and returns its handle, // restoring the caller's current netns before returning. func CreateNetNSWithoutSwitching(name string) (netns.NsHandle, error) { @@ -587,6 +683,9 @@ func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { if err := netlink.AddrReplace(hostLink, HostVethAddr); err != nil { return fmt.Errorf("while assigning host veth address: %w", err) } + if err := netlink.AddrReplace(hostLink, HostVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning host veth ipv6 address: %w", err) + } if err := netlink.LinkSetUp(hostLink); err != nil { return fmt.Errorf("while bringing up host veth: %w", err) } diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index b9c8ac45f..d74442da6 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/nftables" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" + "golang.org/x/sys/unix" ) // withTestNetNS runs fn with the calling thread inside a throwaway netns @@ -86,6 +87,26 @@ func requireNftables(t *testing.T) { } } +// actorNftTableExists reports whether the actor table is present in the family +// InstallActorNftablesRules creates it in. The family is load-bearing: +// ListTablesOfFamily puts it in the netlink dump header, so the kernel filters +// the dump and a query for the wrong family comes back empty rather than +// erroring. +func actorNftTableExists(t *testing.T) bool { + t.Helper() + c := &nftables.Conn{} + tables, err := c.ListTablesOfFamily(nftables.TableFamilyINet) + if err != nil { + t.Fatalf("listing inet nftables tables: %v", err) + } + for _, table := range tables { + if table.Name == ActorNftTableName { + return true + } + } + return false +} + // linkByName returns the link, or nil when it does not exist. func linkByName(t *testing.T, name string) netlink.Link { t.Helper() @@ -115,6 +136,32 @@ func hasAddr(t *testing.T, link netlink.Link, cidr string) bool { return false } +// assertIPv6AddrNoDAD requires cidr to be present on link and to carry +// IFA_F_NODAD. +// +// The flag is the whole point: the ateom container is unprivileged, so the +// accept_dad sysctl this replaced could not be written and setup failed outright +// on a real worker. It passes as root, where /proc/sys is writable either way, +// so nothing else here would catch a regression back to the sysctl. +func assertIPv6AddrNoDAD(t *testing.T, link netlink.Link, cidr string) { + t.Helper() + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + t.Fatalf("listing IPv6 addresses of %q: %v", link.Attrs().Name, err) + } + want := MustParseAddr(cidr) + for _, addr := range addrs { + if addr.IPNet == nil || addr.IPNet.String() != want.IPNet.String() { + continue + } + if addr.Flags&unix.IFA_F_NODAD == 0 { + t.Errorf("%s on %q has flags %#x, want IFA_F_NODAD (%#x) set", cidr, link.Attrs().Name, addr.Flags, unix.IFA_F_NODAD) + } + return + } + t.Errorf("%q does not carry %s, got %v", link.Attrs().Name, cidr, addrs) +} + // TestSetupActorNetworkFinalState pins the namespace state gVisor and the // micro-VM guest read after an activation: what links exist, where, with which // addresses and routes. It deliberately asserts the end state rather than the @@ -143,6 +190,7 @@ func TestSetupActorNetworkFinalState(t *testing.T) { if host.Attrs().Flags&1 == 0 { // net.FlagUp t.Errorf("host veth %q is not up", HostVethName) } + assertIPv6AddrNoDAD(t, host, HostVethIPv6CIDR) // The actor interface must exist ONLY in the interior netns. A peer left // in the pod netns would mean the pair was built the old way, and worse, @@ -162,6 +210,7 @@ func TestSetupActorNetworkFinalState(t *testing.T) { if actor.Attrs().Flags&1 == 0 { t.Errorf("actor veth %q is not up", ActorVethName) } + assertIPv6AddrNoDAD(t, actor, ActorVethIPv6CIDR) if lo := linkByName(t, "lo"); lo == nil { t.Error("interior netns has no loopback") @@ -215,9 +264,20 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { if linkByName(t, HostVethName) == nil { t.Fatalf("host veth %q missing after activation %d", HostVethName, i) } + if !actorNftTableExists(t) { + t.Fatalf("nftables table %q missing after activation %d", ActorNftTableName, i) + } if err := CleanupActorNetwork(ctx, interior); err != nil { t.Fatalf("CleanupActorNetwork (activation %d): %v", i, err) } + // Install and teardown have to name the same family. When they do not, + // teardown's dump comes back empty, its "missing tables are already + // clean" path reports success, and the table survives -- so the next + // activation stacks another copy of every chain and rule onto it and + // the leak is invisible to every other assertion here. + if actorNftTableExists(t) { + t.Fatalf("nftables table %q survived cleanup after activation %d", ActorNftTableName, i) + } } // Cleanup is idempotent: the extra call after the loop's last one must @@ -228,6 +288,9 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { if stray := linkByName(t, HostVethName); stray != nil { t.Errorf("host veth %q survived cleanup", HostVethName) } + if actorNftTableExists(t) { + t.Errorf("nftables table %q survived a repeated cleanup", ActorNftTableName) + } if err := NetNSDo(ctx, interior, func(context.Context) error { if stray := linkByName(t, ActorVethName); stray != nil { t.Errorf("actor veth %q survived cleanup", ActorVethName) From aa8e395a3d9f054ef67e41f252e16e864a9534b8 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Tue, 18 Aug 2026 21:55:02 -0700 Subject: [PATCH 09/10] ateomnet: skip the actor veth IPv6 setup when the netns has IPv6 disabled An IPv4-only cluster leaves net.ipv6.conf.all.disable_ipv6=1 in the worker pod netns, which is the default on IPv4-only GKE, and netlink there rejects the veth's IPv6 address with EPERM. The assignment sits on the path of every SetupActorNetwork call site, so actor startup went from working to failing outright and the actor never left ResumeGoldenActor. Gate the IPv6 address and default route on a per-link disable_ipv6 read, leaving the interior IPv4-only on those clusters instead of failing. A root-gated test covers a netns with IPv6 disabled; reverting the gate reproduces the EPERM against it. (cherry picked from commit 209b7ddc501ab6188b98b5f32b22415bfaf60685) --- internal/ateomnet/net.go | 43 ++++++++++--- internal/ateomnet/net_linux_test.go | 99 +++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 10 deletions(-) diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 2e1d27dfa..fab49a349 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -137,8 +137,11 @@ func ConfigureActorVeth(ctx context.Context) error { if err := netlink.AddrReplace(actorLink, ActorVethAddr); err != nil { return fmt.Errorf("while assigning actor veth address: %w", err) } - if err := netlink.AddrReplace(actorLink, ActorVethIPv6Addr); err != nil { - return fmt.Errorf("while assigning actor veth ipv6 address: %w", err) + actorIPv6 := LinkIPv6Enabled(ActorVethName) + if actorIPv6 { + if err := netlink.AddrReplace(actorLink, ActorVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning actor veth ipv6 address: %w", err) + } } if err := netlink.LinkSetUp(actorLink); err != nil { @@ -151,12 +154,14 @@ func ConfigureActorVeth(ctx context.Context) error { }); err != nil { return fmt.Errorf("while installing actor default route: %w", err) } - if err := netlink.RouteReplace(&netlink.Route{ - LinkIndex: actorLink.Attrs().Index, - Gw: ActorVethIPv6GwIP, - Dst: &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}, - }); err != nil { - return fmt.Errorf("while installing actor default ipv6 route: %w", err) + if actorIPv6 { + if err := netlink.RouteReplace(&netlink.Route{ + LinkIndex: actorLink.Attrs().Index, + Gw: ActorVethIPv6GwIP, + Dst: &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}, + }); err != nil { + return fmt.Errorf("while installing actor default ipv6 route: %w", err) + } } return nil @@ -263,6 +268,20 @@ func EnableIPv4Forwarding() error { return nil } +// LinkIPv6Enabled reports whether IPv6 addresses can be assigned to the named +// link in the current netns. An IPv4-only cluster leaves disable_ipv6=1 in the +// worker pod netns — the default on IPv4-only GKE — and netlink then rejects +// every IPv6 address with EPERM; a kernel built without IPv6 has no sysctl at +// all. Either way the actor interior stays IPv4-only rather than failing to +// start. +func LinkIPv6Enabled(name string) bool { + b, err := os.ReadFile("/proc/sys/net/ipv6/conf/" + name + "/disable_ipv6") + if err != nil { + return false + } + return len(b) > 0 && b[0] == '0' +} + // writeSysctlIfUnset writes "1\n" to a sysctl path unless it already reads "1". // If the path does not exist (e.g. IPv6 sysctls on a kernel with IPv6 disabled), // it returns nil — IPv6 forwarding is simply unavailable, not an error. @@ -683,8 +702,12 @@ func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { if err := netlink.AddrReplace(hostLink, HostVethAddr); err != nil { return fmt.Errorf("while assigning host veth address: %w", err) } - if err := netlink.AddrReplace(hostLink, HostVethIPv6Addr); err != nil { - return fmt.Errorf("while assigning host veth ipv6 address: %w", err) + if LinkIPv6Enabled(HostVethName) { + if err := netlink.AddrReplace(hostLink, HostVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning host veth ipv6 address: %w", err) + } + } else { + slog.Info("IPv6 disabled in the worker pod netns; actor networking is IPv4-only", "link", HostVethName) } if err := netlink.LinkSetUp(hostLink); err != nil { return fmt.Errorf("while bringing up host veth: %w", err) diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index d74442da6..3897529f8 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -19,6 +19,7 @@ package ateomnet import ( "context" "errors" + "os" "runtime" "testing" @@ -247,6 +248,104 @@ func TestSetupActorNetworkFinalState(t *testing.T) { }) } +// disableIPv6 turns IPv6 off in the current netns the way an IPv4-only cluster +// does, so a link created afterwards rejects every IPv6 address. +func disableIPv6(t *testing.T) { + t.Helper() + for _, knob := range []string{"all", "default"} { + path := "/proc/sys/net/ipv6/conf/" + knob + "/disable_ipv6" + if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { + t.Skipf("cannot disable IPv6 in this netns (%s): %v", path, err) + } + } +} + +// assertNoIPv6Addr requires link to carry no IPv6 address at all. +func assertNoIPv6Addr(t *testing.T, link netlink.Link) { + t.Helper() + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + t.Fatalf("listing IPv6 addresses of %q: %v", link.Attrs().Name, err) + } + if len(addrs) != 0 { + t.Errorf("%q carries IPv6 addresses %v in an IPv4-only netns, want none", link.Attrs().Name, addrs) + } +} + +// TestSetupActorNetworkIPv4OnlyNetNS covers a worker pod whose netns has IPv6 +// disabled, which is the default on an IPv4-only GKE cluster. Assigning the veth +// its IPv6 address there fails with EPERM, and because that happens on the path +// of every SetupActorNetwork call site an ungated attempt takes actor startup +// from working to totally broken — the actor never leaves ResumeGoldenActor. +// The IPv4 half must come up exactly as it does with IPv6 available. +func TestSetupActorNetworkIPv4OnlyNetNS(t *testing.T) { + roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") + ctx := context.Background() + + withTestNetNS(t, func(interior netns.NsHandle) { + requireNftables(t) + + disableIPv6(t) + if err := NetNSDo(ctx, interior, func(context.Context) error { + disableIPv6(t) + return nil + }); err != nil { + t.Fatalf("disabling IPv6 in the interior netns: %v", err) + } + + if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { + t.Fatalf("SetupActorNetwork on an IPv4-only netns: %v", err) + } + + host := linkByName(t, HostVethName) + if host == nil { + t.Fatalf("host veth %q missing from the pod netns", HostVethName) + } + if !hasAddr(t, host, HostVethCIDR) { + t.Errorf("host veth %q does not carry %s", HostVethName, HostVethCIDR) + } + if host.Attrs().Flags&1 == 0 { // net.FlagUp + t.Errorf("host veth %q is not up", HostVethName) + } + assertNoIPv6Addr(t, host) + + if err := NetNSDo(ctx, interior, func(context.Context) error { + actor := linkByName(t, ActorVethName) + if actor == nil { + t.Fatalf("actor veth %q missing from the interior netns", ActorVethName) + } + if !hasAddr(t, actor, ActorVethCIDR) { + t.Errorf("actor veth %q does not carry %s", ActorVethName, ActorVethCIDR) + } + if actor.Attrs().Flags&1 == 0 { + t.Errorf("actor veth %q is not up", ActorVethName) + } + assertNoIPv6Addr(t, actor) + + routes, err := netlink.RouteList(actor, netlink.FAMILY_V4) + if err != nil { + t.Fatalf("listing interior routes: %v", err) + } + var haveDefault bool + for _, route := range routes { + ones := 0 + if route.Dst != nil { + ones, _ = route.Dst.Mask.Size() + } + if (route.Dst == nil || ones == 0) && route.Gw.Equal(ActorVethGwIP) { + haveDefault = true + } + } + if !haveDefault { + t.Errorf("interior netns has no default route via %s, got %v", ActorVethGateway, routes) + } + return nil + }); err != nil { + t.Fatalf("inspecting interior netns: %v", err) + } + }) +} + // TestSetupActorNetworkIsRepeatable covers the activation cycle a reused worker // runs: set up, tear down, set up again. The second setup has to succeed against // whatever the first one left behind. From e5703c4fef6dcdf14b899b610f96f2dcfca02c37 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Fri, 14 Aug 2026 11:03:44 -0700 Subject: [PATCH 10/10] ci: add an IPv6-only kind e2e job Runs the full install plus the demo and networking e2e suites against a single-stack IPv6-only kind cluster, and asserts the cluster really is v6-only so a green run cannot quietly become a second IPv4 run. It stays out of the e2e-test merge gate, so it reports IPv6 status without being able to block a PR, and it runs on every PR for now so the results are visible; the TODO on the trigger records the intended ci/ipv6 label gate. ubuntu-latest has no IPv6 egress, so the job stands up tayga for NAT64 and points CoreDNS at an upstream resolver through the well-known prefix. DNS64 is scoped to a catch-all server block: synthesizing AAAA over the cluster zones destroys the v6-only ClusterIP answers and the control plane never comes up. (cherry picked from commit ef02488c8738bc5317f57c73e796e6a37141be93) --- .github/workflows/e2e-ipv6.yaml | 370 ++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 .github/workflows/e2e-ipv6.yaml diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml new file mode 100644 index 000000000..308f12360 --- /dev/null +++ b/.github/workflows/e2e-ipv6.yaml @@ -0,0 +1,370 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e-ipv6 +# Separate from pr-workflow.yaml so this can be gated independently -- and so a +# 40-minute IPv6 run never delays that workflow's merge-gating jobs. +# +# TODO(246): gate this before merging. It runs on every PR today so the IPv6-only +# results are visible without a maintainer having to act first; the intended +# steady state is a `ci/ipv6` label, which needs the label created upstream: +# +# on: {pull_request: {types: [labeled, opened, synchronize, reopened]}} +# if: contains(github.event.pull_request.labels.*.name, 'ci/ipv6') +# +# Either way this job stays out of the `e2e-test` gate, so it never blocks a PR. +on: + pull_request: +permissions: + contents: read +jobs: + e2e-test-ipv6: + runs-on: ubuntu-latest + # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's + # 6-hour default. A broken IPv6 cluster does not crash, it misses + # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. + timeout-minutes: 40 + env: + # Non-default name so these steps can be replayed locally without + # touching an existing cluster. install-ate-kind.sh does not derive + # KUBECTL_CONTEXT from the cluster name the way run-e2e-kind.sh does, + # so both have to be set here. + KIND_CLUSTER_NAME: ate-ipv6 + KUBECTL_CONTEXT: kind-ate-ipv6 + # 8.8.8.8 reached through the well-known NAT64 prefix. CoreDNS is a + # v6-only pod on a runner with no IPv6 egress of its own, so this is the + # only shape of upstream resolver it can reach. See "Set up NAT64". + IPV6_DNS_UPSTREAM: 64:ff9b::808:808 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: 'go.mod' + - name: Free disk space + # kind node image + control-plane images + snapshots are tight on the + # ~14GB runner disk even without the micro-VM assets. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Enable IPv6 in the Docker daemon + # ubuntu-latest ships dockerd with IPv6 off, so kind would create its + # network v4-only and create-kind-cluster.sh would reject the cluster. + # Merge the two keys into whatever daemon.json the runner image ships + # rather than replacing the file. + run: | + sudo mkdir -p /etc/docker + [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo cat /etc/docker/daemon.json \ + | jq '. + {"ipv6": true, "ip6tables": true}' \ + | sudo tee /etc/docker/daemon.json.new >/dev/null + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + sudo systemctl restart docker + docker network inspect bridge --format 'bridge EnableIPv6={{.EnableIPv6}}' + - name: Set up NAT64 on the runner + # ubuntu-latest has no IPv6 egress whatsoever -- measured, not assumed: + # every curl -6 fails in ~2ms. A v6-only cluster still has to reach real + # v4 destinations (atelet fetches the gVisor tarball from GCS, + # TestActorEgress fetches example.com), so the runner translates for it. + # Ordered after the dockerd restart, which rebuilds the iptables chains + # these rules live in. + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq tayga dnsutils + # tayga answers to .1/::1; the tun holds .2/::2 so host-originated + # traffic is not sourced from tayga's own address, which is + # self-addressed rather than translatable. The pool avoids both. + sudo tee /etc/tayga.conf >/dev/null <<'EOF' + tun-device nat64 + ipv4-addr 192.168.255.1 + # tayga refuses the well-known prefix with an RFC1918 pool unless it + # also holds a v6 address of its own, outside that prefix. + ipv6-addr 2001:db8:64::1 + prefix 64:ff9b::/96 + dynamic-pool 192.168.255.128/25 + data-dir /var/spool/tayga + EOF + sudo mkdir -p /var/spool/tayga + sudo tayga --mktun + sudo ip link set nat64 up + sudo ip addr add 192.168.255.2/24 dev nat64 + sudo ip -6 addr add 2001:db8:64::2/128 dev nat64 + sudo ip -6 route add 64:ff9b::/96 dev nat64 src 2001:db8:64::2 + sudo sysctl -qw net.ipv4.ip_forward=1 + sudo sysctl -qw net.ipv6.conf.all.forwarding=1 + sudo iptables -t nat -A POSTROUTING -s 192.168.255.0/24 -j MASQUERADE + # Insert, not append: docker sets the FORWARD policy to DROP. + sudo iptables -I FORWARD 1 -i nat64 -j ACCEPT + sudo iptables -I FORWARD 1 -o nat64 -j ACCEPT + sudo ip6tables -I FORWARD 1 -i nat64 -j ACCEPT + sudo ip6tables -I FORWARD 1 -o nat64 -j ACCEPT + # -d keeps tayga in the foreground and logs every dropped packet with a + # reason; detaching hides exactly the failures worth diagnosing. + sudo sh -c 'nohup tayga -d --config /etc/tayga.conf >/tmp/tayga.log 2>&1 &' + sleep 3 + pgrep -a tayga || { + echo "::error::tayga is not running"; sudo cat /tmp/tayga.log; exit 1; + } + - name: Verify NAT64 before building anything on it + # Hard gate. The cluster takes ~4 minutes and every step after it depends + # on translation working, so a broken translator should fail here with + # one clear message rather than as a rollout timeout ten minutes later. + run: | + # Map a live A record rather than hardcoding one: example.com's old + # 93.184.216.34 is retired and would fail for the wrong reason. + v4=$(getent ahostsv4 storage.googleapis.com | awk 'NR==1{print $1}') + # shellcheck disable=SC2086 + set -- ${v4//./ } + v6=$(printf '64:ff9b::%02x%02x:%02x%02x' "$1" "$2" "$3" "$4") + echo "NAT64 maps ${v4} -> ${v6}" + dig +timeout=5 +tries=1 @"${IPV6_DNS_UPSTREAM}" storage.googleapis.com A +short + code=$(curl -6 -sS -m 15 -o /dev/null -w '%{http_code}' \ + --resolve "storage.googleapis.com:443:[${v6}]" \ + https://storage.googleapis.com/ || echo 000) + echo "NAT64 HTTPS probe returned ${code}" + case "${code}" in + # Any HTTP status proves the translator carried a TCP stream; GCS + # answers a bare / with 400. ICMP is separately blocked, so a ping + # test here would report a failure that does not matter. + 2*|3*|4*) ;; + *) echo "::error::NAT64 is not translating; the cluster cannot egress" + sudo cat /tmp/tayga.log || true + exit 1 ;; + esac + - name: Create cluster + env: + IP_FAMILY: ipv6 + run: hack/create-kind-cluster.sh + - name: Assert the cluster is single-stack IPv6 + # This job is worthless if the cluster is not actually v6-only, and a + # green run leaves no evidence either way -- the diagnostics dump below + # only runs on failure. A kind default change or an IP_FAMILY regression + # would otherwise turn this into a second IPv4 run that reports success. + # Checked here rather than later so it fails as itself. + # + # PreferDualStack Services resolving to a single clusterIP is the + # positive signal: on a dual-stack cluster they would get two. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + pod_cidrs=$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}') + svc_ips=$(k -n default get svc kubernetes -o jsonpath='{.spec.clusterIPs[*]}') + node_ips=$(k get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}') + for pair in "podCIDRs=${pod_cidrs}" "kubernetes.clusterIPs=${svc_ips}" "node.InternalIP=${node_ips}"; do + case "${pair#*=}" in + *.*) echo "::error::not single-stack IPv6 -- ${pair}"; exit 1 ;; + "") echo "::error::empty, cannot confirm IP family -- ${pair}"; exit 1 ;; + esac + echo " ${pair}" + done + echo "single-stack IPv6 confirmed" + - name: Apply DNS64 to external names only + # create-kind-cluster.sh already points CoreDNS at IPV6_DNS_UPSTREAM, so + # names resolve -- but the answers are unusable. Plain DNS64 synthesizes + # only for names with no AAAA, and the external names this job needs + # (storage.googleapis.com, example.com) do have AAAA records, pointing at + # real IPv6 addresses the runner cannot reach. Only translate_all forces + # them through the prefix. + # + # translate_all cannot go in the same server block as the cluster zones. + # dns64 wraps the whole plugin chain below it, and it answers a AAAA query + # by synthesizing from A -- so for an AAAA-only name it synthesizes from + # nothing and returns an empty answer. Every ClusterIP on a v6-only + # cluster is AAAA-only, so a single-block Corefile takes out all + # in-cluster service discovery: ate-api-server cannot find + # valkey-cluster.ate-system.svc and the install times out. + # + # So: cluster zones keep the chain kind shipped, the registry gets its own + # block, and dns64 sits in the catch-all with the forwarder. + run: | + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}' > /tmp/Corefile + # Read the registry address out of the Corefile create-kind-cluster.sh + # wrote; docker inspect would need it picked out of the several + # networks the registry is attached to. + reg_v6=$(awk '$2 == "kind-registry" {print $1; exit}' /tmp/Corefile) + if [ -z "${reg_v6}" ]; then + echo "::error::no kind-registry hosts entry in the Corefile" + cat /tmp/Corefile; exit 1 + fi + # Re-zone the block kind shipped and lift out the two directives that + # move elsewhere, keeping health/ready/kubernetes/cache/etc. as-is. + awk ' + NR == 1 && /^\.:53[[:space:]]*\{/ { + print "cluster.local:53 in-addr.arpa:53 ip6.arpa:53 {"; next + } + /^ (hosts|forward)([[:space:]].*)?\{$/ { skip = 1; next } + skip && /^ \}$/ { skip = 0; next } + skip { next } + { print } + ' /tmp/Corefile > /tmp/Corefile.new + if ! grep -q '^cluster.local:53' /tmp/Corefile.new; then + echo "::error::Corefile did not start with the .:53 block kind ships" + cat /tmp/Corefile; exit 1 + fi + if grep -q 'forward' /tmp/Corefile.new; then + echo "::error::the forward block survived the split" + cat /tmp/Corefile.new; exit 1 + fi + cat >>/tmp/Corefile.new < /tmp/coredns-dns64.yaml + kubectl --context="$KUBECTL_CONTEXT" -n kube-system patch cm coredns \ + --type=merge --patch-file /tmp/coredns-dns64.yaml + kubectl --context="$KUBECTL_CONTEXT" -n kube-system rollout restart deploy/coredns + kubectl --context="$KUBECTL_CONTEXT" -n kube-system rollout status deploy/coredns --timeout=120s + cat /tmp/Corefile.new + - name: Verify cluster DNS answers both internal and external names + # The install is the next step and it takes ten minutes to fail. A DNS + # regression is the failure this Corefile is most likely to cause, so + # assert all three cases here where the message is unambiguous. + run: | + set -o pipefail + kubectl --context="$KUBECTL_CONTEXT" run dnscheck --rm --attach --quiet \ + --restart=Never --image=busybox:1.36 --command -- \ + sh -c ' + nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 \ + || { echo "FAIL: an in-cluster Service does not resolve"; exit 1; } + nslookup storage.googleapis.com 2>/dev/null | grep -q "64:ff9b" \ + || { echo "FAIL: external names are not synthesized through NAT64"; exit 1; } + # Informational. Nothing downstream resolves this from a pod -- + # containerd pulls images on the node, and create-kind-cluster.sh + # runs its own registry probe before DNS64 is applied -- so a miss + # here is not a reason to fail the job. Printed because a change + # here would still be worth seeing. + echo "--- kind-registry, informational" + nslookup kind-registry 2>&1 | tail -4 + echo "DNS-OK" + ' | tee /tmp/dnscheck.log + grep -q DNS-OK /tmp/dnscheck.log + - name: Install Agent Substrate + run: hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth=cert + - name: Assert the control plane is up + # install-ate.sh runs under pipefail, so a failed apply does propagate. + # What it would not catch is a Deployment that rolls out and then + # crash-loops. Re-check everything deploy_ate_system waits on -- + # atenet-egress included, since a non-dual-stack Envoy listener fails + # there first, by way of a readiness probe the kubelet cannot reach. + run: | + for r in deployment/ate-api-server deployment/ate-controller \ + deployment/atenet-router deployment/atenet-egress \ + statefulset/valkey-cluster daemonset/atelet; do + kubectl --context="$KUBECTL_CONTEXT" -n ate-system rollout status "$r" --timeout=120s + done + kubectl --context="$KUBECTL_CONTEXT" -n podcertificate-controller-system \ + rollout status deployment/podcertificate-controller --timeout=120s + if kubectl --context="$KUBECTL_CONTEXT" -n ate-system get pods \ + -o jsonpath='{.items[*].status.containerStatuses[*].state.waiting.reason}' \ + | grep -q CrashLoopBackOff; then + echo "::error::a pod in ate-system is in CrashLoopBackOff" + exit 1 + fi + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Assert the demo fixtures exist + # A failed demo deploy exits 0: install-ate.sh dispatches demos through + # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends + # in an unconditional `return 0`. Without this the suites fail later with + # "ActorTemplate not found", pointing at the tests instead of the install. + run: | + for ns_tmpl in ate-demo-counter/counter ate-demo-egress/egress; do + ns=${ns_tmpl%/*}; tmpl=${ns_tmpl#*/} + kubectl --context="$KUBECTL_CONTEXT" -n "${ns}" get actortemplate "${tmpl}" \ + || { echo "::error::${ns_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + # One suite per step: run-e2e.sh takes exactly one target path, and this + # way a failure names the suite that produced it. + - name: Run E2E tests (demo) + id: e2e-demo + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color 2>&1 \ + | tee /tmp/e2e-demo.log + - name: Run E2E tests (networking) + # Runs even when demo failed -- networking is the half most likely to + # expose a single-family bug -- but stays skipped when an earlier step + # left no cluster to test against. always() is required, not decorative: + # an if: without a status function is implicitly ANDed with success(), + # which skips this step on exactly the failure it is meant to survive. + if: always() && steps.e2e-demo.outcome != 'skipped' + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking.log + - name: Guard against a vacuously green run + # A suite that gates on a dual-stack Service and skips itself on v6-only + # exits 0, so a suite that only skipped would otherwise read as a pass. + # The bar is one real PASS per suite, not zero skips: demo legitimately + # skips the micro-VM-only Golden resume and the CSI volume tests on any + # family. Skips are printed so a growing list gets noticed. + # Skipped when the suites never ran: with no logs to count, this step + # would otherwise report a reassuring zero on a job that failed earlier. + if: always() && steps.e2e-demo.outcome != 'skipped' + run: | + for f in /tmp/e2e-demo.log /tmp/e2e-networking.log; do + [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } + passed=$(grep -c -- '--- PASS' "$f" || true) + echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" + grep -h -- '--- SKIP' "$f" || true + if [ "${passed}" -eq 0 ]; then + echo "::error::${f} has no passing tests -- a suite that only skips proves nothing" + exit 1 + fi + done + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context="$KUBECTL_CONTEXT" get actortemplate,workerpool,pods -A -o wide || true + dump() { + echo "=== logs: $1/$2 ===" + kubectl --context="$KUBECTL_CONTEXT" logs -n "$1" "$2" --all-containers --tail=300 2>/dev/null || true + } + for p in $(kubectl --context="$KUBECTL_CONTEXT" get pods -n ate-system -o name 2>/dev/null); do + dump ate-system "$p" + done + # Every worker pod in any namespace: the demo pools plus the e2e suites' + # randomly-named per-test namespaces, which the suites keep on failure. + kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ + -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ + | while read -r ns name; do dump "$ns" "$name"; done + # IPv6-specific: the rewritten Corefile, and what each Service actually + # got assigned, are the two things that differ from the IPv4 job. + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l k8s-app=kube-dns --tail=100 || true + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' || true + kubectl --context="$KUBECTL_CONTEXT" get svc -A \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,POLICY:.spec.ipFamilyPolicy,IPS:.spec.clusterIPs || true + # tayga logs a reason for every packet it declines to translate, which + # is the only view of an egress failure that is not a bare timeout. + echo "=== tayga ===" + sudo tail -100 /tmp/tayga.log || true