diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml new file mode 100644 index 0000000000..c7a0e9a519 --- /dev/null +++ b/.github/workflows/e2e-ipv6.yaml @@ -0,0 +1,366 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e-ipv6 +# Separate from pr-workflow.yaml so this can be gated independently -- and so a +# 40-minute IPv6 run never delays that workflow's merge-gating jobs. +# +# TODO(246): gate this before merging. It runs on every PR today so the IPv6-only +# results are visible without a maintainer having to act first; the intended +# steady state is a `ci/ipv6` label, which needs the label created upstream: +# +# on: {pull_request: {types: [labeled, opened, synchronize, reopened]}} +# if: contains(github.event.pull_request.labels.*.name, 'ci/ipv6') +# +# Either way this job stays out of the `e2e-test` gate, so it never blocks a PR. +on: + pull_request: +permissions: + contents: read +jobs: + e2e-test-ipv6: + runs-on: ubuntu-latest + # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's + # 6-hour default. A broken IPv6 cluster does not crash, it misses + # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. + timeout-minutes: 40 + env: + # Non-default name so these steps can be replayed locally without + # touching an existing cluster. install-ate-kind.sh does not derive + # KUBECTL_CONTEXT from the cluster name the way run-e2e-kind.sh does, + # so both have to be set here. + KIND_CLUSTER_NAME: ate-ipv6 + KUBECTL_CONTEXT: kind-ate-ipv6 + # 8.8.8.8 reached through the well-known NAT64 prefix. CoreDNS is a + # v6-only pod on a runner with no IPv6 egress of its own, so this is the + # only shape of upstream resolver it can reach. See "Set up NAT64". + IPV6_DNS_UPSTREAM: 64:ff9b::808:808 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: 'go.mod' + - name: Free disk space + # kind node image + control-plane images + snapshots are tight on the + # ~14GB runner disk even without the micro-VM assets. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Enable IPv6 in the Docker daemon + # ubuntu-latest ships dockerd with IPv6 off, so kind would create its + # network v4-only and create-kind-cluster.sh would reject the cluster. + # Merge the two keys into whatever daemon.json the runner image ships + # rather than replacing the file. + run: | + sudo mkdir -p /etc/docker + [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo cat /etc/docker/daemon.json \ + | jq '. + {"ipv6": true, "ip6tables": true}' \ + | sudo tee /etc/docker/daemon.json.new >/dev/null + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + sudo systemctl restart docker + docker network inspect bridge --format 'bridge EnableIPv6={{.EnableIPv6}}' + - name: Set up NAT64 on the runner + # ubuntu-latest has no IPv6 egress whatsoever -- measured, not assumed: + # every curl -6 fails in ~2ms. A v6-only cluster still has to reach real + # v4 destinations (atelet fetches the gVisor tarball from GCS, + # TestActorEgress fetches example.com), so the runner translates for it. + # Ordered after the dockerd restart, which rebuilds the iptables chains + # these rules live in. + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq tayga dnsutils + # tayga answers to .1/::1; the tun holds .2/::2 so host-originated + # traffic is not sourced from tayga's own address, which is + # self-addressed rather than translatable. The pool avoids both. + sudo tee /etc/tayga.conf >/dev/null <<'EOF' + tun-device nat64 + ipv4-addr 192.168.255.1 + # tayga refuses the well-known prefix with an RFC1918 pool unless it + # also holds a v6 address of its own, outside that prefix. + ipv6-addr 2001:db8:64::1 + prefix 64:ff9b::/96 + dynamic-pool 192.168.255.128/25 + data-dir /var/spool/tayga + EOF + sudo mkdir -p /var/spool/tayga + sudo tayga --mktun + sudo ip link set nat64 up + sudo ip addr add 192.168.255.2/24 dev nat64 + sudo ip -6 addr add 2001:db8:64::2/128 dev nat64 + sudo ip -6 route add 64:ff9b::/96 dev nat64 src 2001:db8:64::2 + sudo sysctl -qw net.ipv4.ip_forward=1 + sudo sysctl -qw net.ipv6.conf.all.forwarding=1 + sudo iptables -t nat -A POSTROUTING -s 192.168.255.0/24 -j MASQUERADE + # Insert, not append: docker sets the FORWARD policy to DROP. + sudo iptables -I FORWARD 1 -i nat64 -j ACCEPT + sudo iptables -I FORWARD 1 -o nat64 -j ACCEPT + sudo ip6tables -I FORWARD 1 -i nat64 -j ACCEPT + sudo ip6tables -I FORWARD 1 -o nat64 -j ACCEPT + # -d keeps tayga in the foreground and logs every dropped packet with a + # reason; detaching hides exactly the failures worth diagnosing. + sudo sh -c 'nohup tayga -d --config /etc/tayga.conf >/tmp/tayga.log 2>&1 &' + sleep 3 + pgrep -a tayga || { + echo "::error::tayga is not running"; sudo cat /tmp/tayga.log; exit 1; + } + - name: Verify NAT64 before building anything on it + # Hard gate. The cluster takes ~4 minutes and every step after it depends + # on translation working, so a broken translator should fail here with + # one clear message rather than as a rollout timeout ten minutes later. + run: | + # Map a live A record rather than hardcoding one: example.com's old + # 93.184.216.34 is retired and would fail for the wrong reason. + v4=$(getent ahostsv4 storage.googleapis.com | awk 'NR==1{print $1}') + # shellcheck disable=SC2086 + set -- ${v4//./ } + v6=$(printf '64:ff9b::%02x%02x:%02x%02x' "$1" "$2" "$3" "$4") + echo "NAT64 maps ${v4} -> ${v6}" + dig +timeout=5 +tries=1 @"${IPV6_DNS_UPSTREAM}" storage.googleapis.com A +short + code=$(curl -6 -sS -m 15 -o /dev/null -w '%{http_code}' \ + --resolve "storage.googleapis.com:443:[${v6}]" \ + https://storage.googleapis.com/ || echo 000) + echo "NAT64 HTTPS probe returned ${code}" + case "${code}" in + # Any HTTP status proves the translator carried a TCP stream; GCS + # answers a bare / with 400. ICMP is separately blocked, so a ping + # test here would report a failure that does not matter. + 2*|3*|4*) ;; + *) echo "::error::NAT64 is not translating; the cluster cannot egress" + sudo cat /tmp/tayga.log || true + exit 1 ;; + esac + - name: Create cluster + env: + IP_FAMILY: ipv6 + run: hack/create-kind-cluster.sh + - name: Assert the cluster is single-stack IPv6 + # This job is worthless if the cluster is not actually v6-only, and a + # green run leaves no evidence either way -- the diagnostics dump below + # only runs on failure. A kind default change or an IP_FAMILY regression + # would otherwise turn this into a second IPv4 run that reports success. + # Checked here rather than later so it fails as itself. + # + # PreferDualStack Services resolving to a single clusterIP is the + # positive signal: on a dual-stack cluster they would get two. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + pod_cidrs=$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}') + svc_ips=$(k -n default get svc kubernetes -o jsonpath='{.spec.clusterIPs[*]}') + node_ips=$(k get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}') + for pair in "podCIDRs=${pod_cidrs}" "kubernetes.clusterIPs=${svc_ips}" "node.InternalIP=${node_ips}"; do + case "${pair#*=}" in + *.*) echo "::error::not single-stack IPv6 -- ${pair}"; exit 1 ;; + "") echo "::error::empty, cannot confirm IP family -- ${pair}"; exit 1 ;; + esac + echo " ${pair}" + done + echo "single-stack IPv6 confirmed" + - name: Apply DNS64 to external names only + # create-kind-cluster.sh already points CoreDNS at IPV6_DNS_UPSTREAM, so + # names resolve -- but the answers are unusable. Plain DNS64 synthesizes + # only for names with no AAAA, and the external names this job needs + # (storage.googleapis.com, example.com) do have AAAA records, pointing at + # real IPv6 addresses the runner cannot reach. Only translate_all forces + # them through the prefix. + # + # translate_all cannot go in the same server block as the cluster zones. + # dns64 wraps the whole plugin chain below it, and it answers a AAAA query + # by synthesizing from A -- so for an AAAA-only name it synthesizes from + # nothing and returns an empty answer. Every ClusterIP on a v6-only + # cluster is AAAA-only, so a single-block Corefile takes out all + # in-cluster service discovery: ate-api-server cannot find + # valkey-cluster.ate-system.svc and the install times out. + # + # So: cluster zones keep the chain kind shipped, the registry keeps the + # block create-kind-cluster.sh gave it, and dns64 sits in the catch-all + # with the forwarder. + run: | + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}' > /tmp/Corefile + # Re-zone the block kind shipped and lift out its forwarder, which moves + # to the catch-all below; health/ready/kubernetes/cache stay as-is. The + # rules are gated on "first" so they stop at that block's closing brace + # and leave the registry's own block untouched. + awk ' + NR == 1 && /^\.:53[[:space:]]*\{/ { + print "cluster.local:53 in-addr.arpa:53 ip6.arpa:53 {"; first = 1; next + } + first && /^ forward([[:space:]].*)?\{$/ { skip = 1; next } + first && skip && /^ \}$/ { skip = 0; next } + first && skip { next } + first && /^\}$/ { first = 0 } + { print } + ' /tmp/Corefile > /tmp/Corefile.new + if ! grep -q '^cluster.local:53' /tmp/Corefile.new; then + echo "::error::Corefile did not start with the .:53 block kind ships" + cat /tmp/Corefile; exit 1 + fi + # create-kind-cluster.sh owns this block. If it ever goes back to a + # hosts entry inside .:53, the split above silently drops the registry. + if ! grep -q '^kind-registry:53' /tmp/Corefile.new; then + echo "::error::no kind-registry server block in the Corefile" + cat /tmp/Corefile; exit 1 + fi + if grep -q 'forward' /tmp/Corefile.new; then + echo "::error::the forward block survived the split" + cat /tmp/Corefile.new; exit 1 + fi + cat >>/tmp/Corefile.new < /tmp/coredns-dns64.yaml + kubectl --context="$KUBECTL_CONTEXT" -n kube-system patch cm coredns \ + --type=merge --patch-file /tmp/coredns-dns64.yaml + kubectl --context="$KUBECTL_CONTEXT" -n kube-system rollout restart deploy/coredns + kubectl --context="$KUBECTL_CONTEXT" -n kube-system rollout status deploy/coredns --timeout=120s + cat /tmp/Corefile.new + - name: Verify cluster DNS answers both internal and external names + # The install is the next step and it takes ten minutes to fail. A DNS + # regression is the failure this Corefile is most likely to cause, so + # assert all three cases here where the message is unambiguous. + run: | + set -o pipefail + kubectl --context="$KUBECTL_CONTEXT" run dnscheck --rm --attach --quiet \ + --restart=Never --image=busybox:1.36 --command -- \ + sh -c ' + nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 \ + || { echo "FAIL: an in-cluster Service does not resolve"; exit 1; } + nslookup storage.googleapis.com 2>/dev/null | grep -q "64:ff9b" \ + || { echo "FAIL: external names are not synthesized through NAT64"; exit 1; } + # Informational. Nothing downstream resolves this from a pod -- + # containerd pulls images on the node, and create-kind-cluster.sh + # runs its own registry probe before DNS64 is applied -- so a miss + # here is not a reason to fail the job. Printed because a change + # here would still be worth seeing. + echo "--- kind-registry, informational" + nslookup kind-registry 2>&1 | tail -4 + echo "DNS-OK" + ' | tee /tmp/dnscheck.log + grep -q DNS-OK /tmp/dnscheck.log + - name: Install Agent Substrate + run: hack/install-ate-kind.sh --deploy-ate-system + - name: Assert the control plane is up + # install-ate.sh runs under pipefail, so a failed apply does propagate. + # What it would not catch is a Deployment that rolls out and then + # crash-loops. Re-check everything deploy_ate_system waits on -- + # atenet-egress included, since a non-dual-stack Envoy listener fails + # there first, by way of a readiness probe the kubelet cannot reach. + run: | + for r in deployment/ate-api-server deployment/ate-controller \ + deployment/atenet-router deployment/atenet-egress \ + statefulset/valkey-cluster daemonset/atelet; do + kubectl --context="$KUBECTL_CONTEXT" -n ate-system rollout status "$r" --timeout=120s + done + kubectl --context="$KUBECTL_CONTEXT" -n podcertificate-controller-system \ + rollout status deployment/podcertificate-controller --timeout=120s + if kubectl --context="$KUBECTL_CONTEXT" -n ate-system get pods \ + -o jsonpath='{.items[*].status.containerStatuses[*].state.waiting.reason}' \ + | grep -q CrashLoopBackOff; then + echo "::error::a pod in ate-system is in CrashLoopBackOff" + exit 1 + fi + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Assert the demo fixtures exist + # A failed demo deploy exits 0: install-ate.sh dispatches demos through + # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends + # in an unconditional `return 0`. Without this the suites fail later with + # "ActorTemplate not found", pointing at the tests instead of the install. + run: | + for ns_tmpl in ate-demo-counter/counter ate-demo-egress/egress; do + ns=${ns_tmpl%/*}; tmpl=${ns_tmpl#*/} + kubectl --context="$KUBECTL_CONTEXT" -n "${ns}" get actortemplate "${tmpl}" \ + || { echo "::error::${ns_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + # One suite per step: run-e2e.sh takes exactly one target path, and this + # way a failure names the suite that produced it. + - name: Run E2E tests (demo) + id: e2e-demo + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color 2>&1 \ + | tee /tmp/e2e-demo.log + - name: Run E2E tests (networking) + # Runs even when demo failed -- networking is the half most likely to + # expose a single-family bug -- but stays skipped when an earlier step + # left no cluster to test against. always() is required, not decorative: + # an if: without a status function is implicitly ANDed with success(), + # which skips this step on exactly the failure it is meant to survive. + if: always() && steps.e2e-demo.outcome != 'skipped' + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking.log + - name: Guard against a vacuously green run + # A suite that gates on a dual-stack Service and skips itself on v6-only + # exits 0, so a suite that only skipped would otherwise read as a pass. + # The bar is one real PASS per suite, not zero skips: demo legitimately + # skips the micro-VM-only Golden resume and the CSI volume tests on any + # family. Skips are printed so a growing list gets noticed. + # Skipped when the suites never ran: with no logs to count, this step + # would otherwise report a reassuring zero on a job that failed earlier. + if: always() && steps.e2e-demo.outcome != 'skipped' + run: | + for f in /tmp/e2e-demo.log /tmp/e2e-networking.log; do + [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } + passed=$(grep -c -- '--- PASS' "$f" || true) + echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" + grep -h -- '--- SKIP' "$f" || true + if [ "${passed}" -eq 0 ]; then + echo "::error::${f} has no passing tests -- a suite that only skips proves nothing" + exit 1 + fi + done + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context="$KUBECTL_CONTEXT" get actortemplate,workerpool,pods -A -o wide || true + dump() { + echo "=== logs: $1/$2 ===" + kubectl --context="$KUBECTL_CONTEXT" logs -n "$1" "$2" --all-containers --tail=300 2>/dev/null || true + } + for p in $(kubectl --context="$KUBECTL_CONTEXT" get pods -n ate-system -o name 2>/dev/null); do + dump ate-system "$p" + done + # Every worker pod in any namespace: the demo pools plus the e2e suites' + # randomly-named per-test namespaces, which the suites keep on failure. + kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ + -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ + | while read -r ns name; do dump "$ns" "$name"; done + # IPv6-specific: the rewritten Corefile, and what each Service actually + # got assigned, are the two things that differ from the IPv4 job. + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l k8s-app=kube-dns --tail=100 || true + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' || true + kubectl --context="$KUBECTL_CONTEXT" get svc -A \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,POLICY:.spec.ipFamilyPolicy,IPS:.spec.clusterIPs || true + # tayga logs a reason for every packet it declines to translate, which + # is the only view of an egress failure that is not a bare timeout. + echo "=== tayga ===" + sudo tail -100 /tmp/tayga.log || true diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index bd9f2abfc4..ff886dcaaa 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -36,6 +36,8 @@ type dataplaneHealthCheck struct { // untouched, so atunnel always authorizes by the actor's own DNS name -- // ingress.New needs no per-dataplane routing mode. +// healthCheck dials IPv4 loopback, which is why the admin socket in +// manifests/ate-install/atenet-router.yaml needs ipv4_compat. func (r atenetRouter) healthCheck() dataplaneHealthCheck { switch r { case atenetRouterEnvoy: diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index e2d76d8bf9..903ac1fffb 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -1108,6 +1108,27 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { } } +// dualStackAdditionalAddresses returns the IPv6 half of a dual-stack ingress +// listener, to pair with a primary 0.0.0.0 socket on the same port. Ipv4Compat +// stays false: clearing IPV6_V6ONLY would collide with that primary. +func dualStackAdditionalAddresses(port uint32) []*listenerv3.AdditionalAddress { + return []*listenerv3.AdditionalAddress{ + { + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Address: "::", + Ipv4Compat: false, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + }, + } +} + func (x *XdsServer) buildListener() *listenerv3.Listener { hcm := x.buildHcm("ingress_http", true) @@ -1123,6 +1144,7 @@ func (x *XdsServer) buildListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.ingressPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -1182,6 +1204,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.httpsPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -1213,6 +1236,7 @@ func (x *XdsServer) buildConnectTerminateListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.connectPlainTextPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -1246,6 +1270,7 @@ func (x *XdsServer) buildConnectTerminateTLSListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.connectTLSPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 6fa5c428b7..ea39fb7c4d 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -48,6 +48,36 @@ import ( "github.com/agent-substrate/substrate/internal/atunnel" ) +// assertDualStackIngress checks an ingress listener keeps its 0.0.0.0 primary +// and gains exactly one "::" socket on the same port. +func assertDualStackIngress(t *testing.T, l *listenerv3.Listener, wantPort uint32) { + t.Helper() + + sa := l.GetAddress().GetSocketAddress() + if sa.GetAddress() != "0.0.0.0" { + t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) + } + if sa.GetPortValue() != wantPort { + t.Errorf("Expected port %d, got %d", wantPort, sa.GetPortValue()) + } + + addrs := l.GetAdditionalAddresses() + if len(addrs) != 1 { + t.Fatalf("Expected 1 additional address on %s, got %d", l.GetName(), len(addrs)) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Error("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != wantPort { + t.Errorf("Expected additional port %d, got %d", wantPort, asa.GetPortValue()) + } +} + func TestXdsServer_UpdateSnapshot(t *testing.T) { server := NewXdsServer(18000) server.SetConfig(8081, 50052, "10.0.0.1") @@ -150,14 +180,7 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if raw, exists := listenersMap[IngressHTTPListener]; !exists { t.Errorf("Listener name '%s' is missing from snapshot listeners", IngressHTTPListener) } else { - l := raw.(*listenerv3.Listener) - sa := l.GetAddress().GetSocketAddress() - if sa.GetPortValue() != 8081 { - t.Errorf("Expected port 8081, got %d", sa.GetPortValue()) - } - if sa.GetAddress() != "0.0.0.0" { - t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) - } + assertDualStackIngress(t, raw.(*listenerv3.Listener), 8081) } } @@ -192,10 +215,7 @@ func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { t.Errorf("Listener name '%s' is missing from snapshot listeners", IngressHTTPSListener) } else { l := raw.(*listenerv3.Listener) - sa := l.GetAddress().GetSocketAddress() - if sa.GetPortValue() != 8443 { - t.Errorf("Expected port 8443, got %d", sa.GetPortValue()) - } + assertDualStackIngress(t, l, 8443) // Verify the TLS config references the serving cert via SDS rather // than embedding it: inline filename DataSources are read only once @@ -354,16 +374,14 @@ func TestXdsServer_UpdateSnapshot_WithConnect(t *testing.T) { } if raw, exists := listenersMap["connect_terminate"]; !exists { t.Error("connect_terminate listener missing") - } else if sa := raw.(*listenerv3.Listener).GetAddress().GetSocketAddress(); sa.GetPortValue() != 8081 { - t.Errorf("Expected connect_terminate port 8081, got %d", sa.GetPortValue()) + } else { + assertDualStackIngress(t, raw.(*listenerv3.Listener), 8081) } if raw, exists := listenersMap["connect_terminate_tls"]; !exists { t.Error("connect_terminate_tls listener missing") } else { l := raw.(*listenerv3.Listener) - if sa := l.GetAddress().GetSocketAddress(); sa.GetPortValue() != 8444 { - t.Errorf("Expected connect_terminate_tls port 8444, got %d", sa.GetPortValue()) - } + assertDualStackIngress(t, l, 8444) ts := l.GetFilterChains()[0].GetTransportSocket() if ts.GetName() != "envoy.transport_sockets.tls" { t.Errorf("Expected connect_terminate_tls to be TLS-wrapped, got transport socket %q", ts.GetName()) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 0620c09e3f..9e77aae2f2 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -65,8 +65,11 @@ var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") // TODO(liorlieberman) have a sub package for all atunnel releated things like that - atunnelListenAddress = pflag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS") - atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", "0.0.0.0:444", "Address for actor ingress mTLS CONNECT") + // + // Every listen address here is an unspecified wildcard, which Go binds as a + // dual-stack socket. + atunnelListenAddress = pflag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") + atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = pflag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") atunnelClientIdentity = pflag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 6613fcd9b3..d4dde4680c 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -71,8 +71,10 @@ var ( otlpRelaySocket = flag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") - atunnelListenAddress = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS") - atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", "0.0.0.0:444", "Address for actor ingress mTLS CONNECT") + // Every listen address here is an unspecified wildcard, which Go binds as a + // dual-stack socket. + atunnelListenAddress = flag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") + atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") atunnelClientIdentity = flag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 3015fed3b3..329a79293f 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -1065,6 +1065,10 @@ func tailString(s string, n int) string { // agent: configure eth0 (IP/MAC/MTU), install the connected + default routes, and // pin the gateway's ARP entry to its fixed MAC (so a restored guest's frozen // neighbor entry stays valid). +// +// TODO(#246): the guest is configured IPv4-only, so a micro-VM actor sees no +// IPv6 even on a dual-stack pod where the host veth has one. gVisor reads the +// interior netns and picks the address up; this path has to be told. func (s *AteomService) configureGuestNetwork(ctx context.Context, ac *kata.AgentClient, mtu uint64) error { if err := ac.UpdateInterface(ctx, &agentpb.Interface{ Device: ateomnet.ActorVethName, diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index f413e5c953..c2ffdab58b 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -196,6 +199,61 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry +# +# CoreDNS runs dnsPolicy: Default, inheriting the node's IPv4 resolver, which +# no pod here can reach, so external lookups SERVFAIL. Step 3's registry +# wiring is node-side, so it misses atelet too: that pull runs in atelet's own +# netns, where "kind-registry" NXDOMAINs. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + search="forward . /etc/resolv.conf" + replace="forward . ${IPV6_DNS_UPSTREAM}" + # $search unquoted: bash 3.2 splices the quotes in literally. Replacing just + # the target leaves kind's trailing "{ max_concurrent 1000 }" in place. + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # Its own server block, not a hosts entry in .:53. A query is served by the + # one block whose zone is its longest suffix, so only "${reg_name}" arrives + # here -- which is why this hosts needs no fallthrough to avoid NXDOMAINing + # every other name. + patched="${patched} +${reg_name}:53 { + errors + hosts { + ${reg_v6} ${reg_name} + } +}" + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Its own script so it can be re-run against a live cluster: the hosts entry + # above is a snapshot of an address the registry can move off (#1049). + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" REG_NAME="${reg_name}" \ + IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM}" "${ROOT}"/hack/verify-ipv6-dns.sh +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat <&2 + exit 1 + ;; + esac +fi + +# Best-effort: only used to make the registry failure message actionable. +reg_v6="$(docker inspect "${REG_NAME}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)" +reg_at="${reg_v6:+ at [${reg_v6}]:5000}" + +echo "Verifying DNS from a pod..." +# Probe from a pod, not the node: the node is dual-stack and passes either way. +# The registry leg fetches rather than resolves -- the hosts entry is AAAA-only, +# which fails nslookup's A query but satisfies getaddrinfo. +# +# One stream carries every leg and only the last one's exit status, so each leg +# reports a marker on stdout and no failure message may contain one; PROBE_RAN +# and PROBE_DONE bracket the run so a short read is told apart from a leg that +# failed. Read the log once the pod has terminated rather than attaching to it: +# an attach can drop the tail, and a lost registry marker then reads as an +# unreachable registry. Retry the pod, not the query: one that asks before +# CoreDNS settles stays broken for ~30s, while a fresh pod 10s later resolves +# first try. +probe="" +probe_max=4 +for ((probe_attempt = 1; probe_attempt <= probe_max; probe_attempt++)); do + probe_pod="coredns-probe-$$-${probe_attempt}" + kubectl --context="${KUBECTL_CONTEXT}" run "${probe_pod}" \ + --restart=Never --image=busybox:1.36 --command -- \ + sh -c "echo PROBE_RAN + if out=\$(nslookup storage.googleapis.com 2>&1); then + echo RESOLVE_OK + else + echo \"resolve failed: \$(echo \"\$out\" | tail -2 | tr '\n' ' ')\" + fi + if out=\$(wget -T10 -O/dev/null http://${REG_NAME}:5000/v2/ 2>&1); then + echo REGISTRY_OK + else + echo \"registry fetch failed: \$(echo \"\$out\" | tail -1)\" + fi + echo PROBE_DONE" >/dev/null || true + for ((probe_wait = 0; probe_wait < 120; probe_wait++)); do + phase="$(kubectl --context="${KUBECTL_CONTEXT}" get pod "${probe_pod}" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true)" + [[ "${phase}" == "Succeeded" || "${phase}" == "Failed" ]] && break + sleep 1 + done + attempt_out="$(kubectl --context="${KUBECTL_CONTEXT}" logs "${probe_pod}" 2>/dev/null || true)" + kubectl --context="${KUBECTL_CONTEXT}" delete pod "${probe_pod}" \ + --now --ignore-not-found --wait=false >/dev/null 2>&1 || true + # A pod that never started must not bury an earlier one's real failure. + if [[ "${attempt_out}" == *PROBE_RAN* ]]; then probe="${attempt_out}"; fi + # Only the resolve leg is a settling race; a down registry will not fix + # itself, so a finished probe is a verdict either way. An unfinished one + # reported no registry result at all, which is not the same as a failure. + if [[ "${probe}" == *RESOLVE_OK* ]] && + [[ "${probe}" == *REGISTRY_OK* || "${probe}" == *PROBE_DONE* ]]; then + break + fi + if ((probe_attempt < probe_max)); then + echo " the probe did not come back clean; re-probing (attempt $((probe_attempt + 1)) of ${probe_max})..." + sleep 10 + fi +done +if [[ "${probe}" != *RESOLVE_OK* || "${probe}" != *REGISTRY_OK* ]]; then + if [[ "${probe}" != *PROBE_RAN* ]]; then + echo "error: the probe pod never ran, so CoreDNS is unverified" >&2 + echo " check that it scheduled and that 'busybox:1.36' pulled" >&2 + elif [[ "${probe}" != *RESOLVE_OK* ]]; then + echo "error: a pod cannot resolve an external name" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + elif [[ "${probe}" != *PROBE_DONE* ]]; then + echo "error: the probe stopped early, so the registry leg is unverified" >&2 + echo " re-run this script; DNS itself answered" >&2 + else + echo "error: DNS works but a pod cannot reach '${REG_NAME}'${reg_at}" >&2 + echo " check the registry container is up and on the 'kind' network" >&2 + fi + if [[ -n "${probe}" ]]; then + echo " probe output was:" >&2 + printf '%s\n' "${probe}" | sed 's/^/ /' >&2 + fi + exit 1 +fi diff --git a/hack/verify/atenet-admin-bind.sh b/hack/verify/atenet-admin-bind.sh new file mode 100755 index 0000000000..86090c0168 --- /dev/null +++ b/hack/verify/atenet-admin-bind.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dropping ipv4_compat from a gateway's Envoy admin socket fails silently: the +# drain sequence reads the refused IPv4 loopback dial as "Envoy already exited" +# and reports a drain it never performed. No Go test reads these manifests. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +rc=0 +for f in manifests/ate-install/atenet-router.yaml manifests/ate-install/atenet-egress.yaml; do + block="$(grep -A 6 -E '^ *admin:$' "${f}" || true)" + if [[ -z "${block}" ]]; then + echo "${f}: no Envoy admin block found; this check needs updating" >&2 + rc=1 + elif ! grep -q '"::"' <<<"${block}"; then + echo "${f}: Envoy admin socket does not bind \"::\"; an IPv6-primary pod cannot be probed" >&2 + rc=1 + elif ! grep -q 'ipv4_compat: true' <<<"${block}"; then + echo "${f}: Envoy admin socket binds \"::\" without ipv4_compat; IPv4 loopback dials will be refused" >&2 + rc=1 + fi +done + +exit "${rc}" diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 91203a8e04..9ad6997009 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -43,6 +43,22 @@ const ( ActorVethIP = "169.254.17.2" ActorNftTableName = "ateom_actor" + // podPrimaryIfaceName is the worker pod's own interface, the one the CNI + // gave it. It is not ActorVethName despite the identical value: that one + // names the actor's end of the veth, which lives in the interior netns. + podPrimaryIfaceName = "eth0" + + // The IPv6 counterparts of the point-to-point pair above, chosen to echo the + // v4 addresses digit for digit. A fixed ULA rather than an RFC 4193 random + // prefix so the pair stays as readable in a packet dump as 169.254.17.x, and + // not fe80::/10 because a link-local source would need a scope id everywhere + // it is used. It must not overlap the cluster's pod CIDR; kind's dual-stack + // default is fd00:10:244::/56. + HostVethIPv6CIDR = "fd00:169:254::1/126" + ActorVethIPv6CIDR = "fd00:169:254::2/126" + ActorVethIPv6Gateway = "fd00:169:254::1" + ActorVethIPv6IP = "fd00:169:254::2" + // ActorVethSubnet is the point-to-point /30 the actor veth lives on. ActorVethSubnet = "169.254.17.0/30" ) @@ -51,6 +67,10 @@ var ( HostVethAddr = MustParseAddr(HostVethCIDR) ActorVethAddr = MustParseAddr(ActorVethCIDR) ActorVethGwIP = MustParseIP(ActorVethGateway) + + HostVethIPv6Addr = mustParseNoDADAddr(HostVethIPv6CIDR) + ActorVethIPv6Addr = mustParseNoDADAddr(ActorVethIPv6CIDR) + ActorVethIPv6GwIP = MustParseIPv6(ActorVethIPv6Gateway) ) // MustParseAddr parses a CIDR string into a netlink.Addr, panicking on error. @@ -62,6 +82,18 @@ func MustParseAddr(cidr string) *netlink.Addr { return a } +// mustParseNoDADAddr parses a CIDR into an address flagged IFA_F_NODAD. +// +// Per-address flag rather than the interface-wide accept_dad sysctl because the +// ateom container is unprivileged, so containerd mounts /proc/sys read-only. +// DAD is pointless on a point-to-point veth nobody else can reach, and it would +// otherwise hold the address tentative for ~1s on every resume. +func mustParseNoDADAddr(cidr string) *netlink.Addr { + a := MustParseAddr(cidr) + a.Flags = unix.IFA_F_NODAD + return a +} + // MustParseIP parses an IPv4 string into a net.IP, panicking on error. func MustParseIP(s string) net.IP { ip := net.ParseIP(s).To4() @@ -71,6 +103,57 @@ func MustParseIP(s string) net.IP { return ip } +// MustParseIPv6 parses an IPv6 string into a net.IP, panicking on error. An +// IPv4 string is an error: net.IP holds it as a 16-byte v4-mapped address, so +// it would pass a length check and then compare against no IPv6 header. +func MustParseIPv6(s string) net.IP { + ip := net.ParseIP(s) + if ip == nil || ip.To4() != nil { + panic(fmt.Sprintf("parsing constant IPv6 %q", s)) + } + return ip.To16() +} + +// linkIPv6Enabled reports whether IPv6 addresses can be assigned to the named +// link in the current netns. It answers a kernel capability question, not a +// cluster one: IPv4-only GKE leaves disable_ipv6=1 and netlink then rejects +// every IPv6 address with EPERM, but IPv4-only kind leaves it at 0 because the +// node kernel has IPv6 compiled in. A kernel built without IPv6 has no sysctl +// at all. Pair it with linkHasGlobalIPv6 to decide whether the actor gets IPv6; +// on its own it says yes on clusters that have no IPv6 anywhere. +func linkIPv6Enabled(name string) bool { + b, err := os.ReadFile("/proc/sys/net/ipv6/conf/" + name + "/disable_ipv6") + if err != nil { + return false + } + return len(b) > 0 && b[0] == '0' +} + +// linkHasGlobalIPv6 reports whether link carries a global IPv6 address. Called +// on the worker pod's own interface, that is what decides the families the +// actor can egress on. +// +// It answers whether the pod has an address to egress from, not whether that +// address routes anywhere: IsGlobalUnicast is true for a ULA, and dual-stack +// kind hands pods a ULA with no path off the host. Reachability is the +// cluster's problem, not something this can decide from inside the netns. +func linkHasGlobalIPv6(ctx context.Context, link netlink.Link) bool { + // netlink can report ErrDumpInterrupted alongside a valid partial answer. + // Trust a positive result either way: reporting false on a dual-stack pod + // silently strands the actor on IPv4, which is the costlier mistake. + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + slog.WarnContext(ctx, "listing IPv6 addresses of the worker pod interface", + "link", link.Attrs().Name, "error", err, "addressesRead", len(addrs)) + } + for _, addr := range addrs { + if addr.IP.IsGlobalUnicast() { + return true + } + } + return false +} + // MustParseMAC parses a MAC address string into a net.HardwareAddr, panicking on error. func MustParseMAC(s string) net.HardwareAddr { m, err := net.ParseMAC(s) @@ -82,7 +165,9 @@ func MustParseMAC(s string) net.HardwareAddr { // ConfigureActorVeth configures the actor veth inside the interior netns. // It assumes it is already running inside the target network namespace. -func ConfigureActorVeth(ctx context.Context) error { +// ipv6 comes from SetupActorNetwork, which decides it in the worker pod netns; +// this namespace cannot answer the question for itself. +func ConfigureActorVeth(ctx context.Context, ipv6 bool) error { // Run inside the gVisor interior netns. SetupActorNetwork has already created // the veth peer here, under its final name, so this only has to address it. // gVisor reads link names, addresses, and routes from this namespace when the @@ -107,6 +192,12 @@ func ConfigureActorVeth(ctx context.Context) error { if err := netlink.AddrReplace(actorLink, ActorVethAddr); err != nil { return fmt.Errorf("while assigning actor veth address: %w", err) } + if ipv6 { + if err := netlink.AddrReplace(actorLink, ActorVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning actor veth ipv6 address: %w", err) + } + } + if err := netlink.LinkSetUp(actorLink); err != nil { return fmt.Errorf("while bringing up actor veth: %w", err) } @@ -117,6 +208,15 @@ func ConfigureActorVeth(ctx context.Context) error { }); err != nil { return fmt.Errorf("while installing actor default route: %w", err) } + if ipv6 { + if err := netlink.RouteReplace(&netlink.Route{ + LinkIndex: actorLink.Attrs().Index, + Gw: ActorVethIPv6GwIP, + Dst: &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}, + }); err != nil { + return fmt.Errorf("while installing actor default ipv6 route: %w", err) + } + } return nil } @@ -173,7 +273,7 @@ func PodIPv4() (net.IP, error) { // Resolve the worker pod IPv4 address from the pod namespace's real eth0. // Because eth0 now stays in the pod namespace, this IP remains available for // both normal worker connectivity and the temporary inbound DNAT rule. - eth0Link, err := netlink.LinkByName("eth0") + eth0Link, err := netlink.LinkByName(podPrimaryIfaceName) if err != nil { return nil, fmt.Errorf("while getting pod eth0: %w", err) } @@ -192,8 +292,11 @@ func PodIPv4() (net.IP, error) { return nil, fmt.Errorf("pod eth0 has no IPv4 address") } -// EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. -func EnableIPv4Forwarding() error { +// EnableForwarding enables IPv4 and IPv6 forwarding in the current network +// namespace, so actor traffic (including DNS queries on IPv6-capable clusters) +// is routed between the veth and eth0 instead of being dropped by ip_forward() +// or ip6_forward(). +func EnableForwarding() error { // Forwarding is required because actor packets now enter the worker pod via // the host-side veth and then leave through the pod's eth0. Without this, the // kernel would not route traffic between those interfaces even though both @@ -203,20 +306,47 @@ func EnableIPv4Forwarding() error { // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag // is not locked: clear it, write the sysctl, restore ro. const path = "/proc/sys/net/ipv4/ip_forward" + if err := writeSysctlIfUnset(path); err != nil { + return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) + } + // IPv6 forwarding: actor packets that arrive on the veth and leave via eth0 + // are IPv6 on dual-stack / IPv6-only clusters. Without + // net.ipv6.conf.all.forwarding the kernel drops every IPv6 packet in + // ip6_forward(), including the actor's DNS queries. conf.all.forwarding=1 + // also implies the per-interface default, so a single write covers the veth + // and eth0. + const v6path = "/proc/sys/net/ipv6/conf/all/forwarding" + if err := writeSysctlIfUnset(v6path); err != nil { + return fmt.Errorf("while enabling IPv6 forwarding in worker pod netns: %w", err) + } + return nil +} + +// writeSysctlIfUnset writes "1\n" to a sysctl path unless it already reads "1". +// If the path does not exist (e.g. IPv6 sysctls on a kernel with IPv6 disabled), +// it returns nil — IPv6 forwarding is simply unavailable, not an error. +func writeSysctlIfUnset(path string) error { if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { return nil } if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { return nil } + if _, err := os.Stat(path); os.IsNotExist(err) { + // Path absent (e.g. IPv6 disabled in kernel): nothing to enable. + return nil + } + // Without privileged, the container runtime bind-mounts /proc/sys read-only. + // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag + // is not locked: clear it, write the sysctl, restore ro. if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { - return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err) + return fmt.Errorf("while remounting /proc/sys read-write to enable forwarding: %w", err) } defer func() { _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") }() if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { - return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) + return fmt.Errorf("while writing %s: %w", path, err) } return nil } @@ -229,9 +359,6 @@ func InstallActorNftablesRules(egressPort uint16) error { // rules in an ateom-owned table makes cleanup simple and avoids mutating // Kubernetes or CNI-managed chains directly. // - // TODO: Add IPv6 veth addressing, forwarding, and nftables rules once actor - // networking supports dual-stack pods. The current actor network is IPv4-only. - // // The rules do three things: // // * prerouting: redirect new actor TCP connections to atunnel's local @@ -248,7 +375,7 @@ func InstallActorNftablesRules(egressPort uint16) error { c := &nftables.Conn{} table := &nftables.Table{ - Family: nftables.TableFamilyIPv4, + Family: nftables.TableFamilyINet, Name: ActorNftTableName, } c.AddTable(table) @@ -263,6 +390,9 @@ func InstallActorNftablesRules(egressPort uint16) error { if redirectRule := ActorEgressRedirectRule(table, prerouting, egressPort); redirectRule != nil { c.AddRule(redirectRule) } + if redirectRuleIPv6 := ActorIPv6EgressRedirectRule(table, prerouting, egressPort); redirectRuleIPv6 != nil { + c.AddRule(redirectRuleIPv6) + } postrouting := c.AddChain(&nftables.Chain{ Name: "postrouting", @@ -276,6 +406,11 @@ func InstallActorNftablesRules(egressPort uint16) error { Chain: postrouting, Exprs: append(IPSourceEqual(ActorVethIP), &expr.Masq{}), }) + c.AddRule(&nftables.Rule{ + Table: table, + Chain: postrouting, + Exprs: append(IPv6SourceEqual(ActorVethIPv6IP), &expr.Masq{}), + }) acceptPolicy := nftables.ChainPolicyAccept forward := c.AddChain(&nftables.Chain{ @@ -305,20 +440,26 @@ func RemoveActorNftablesRules() error { // Delete the whole ateom nftables table if it exists. The table is // per-worker and currently per-active-actor because this worker path runs at // most one actor at a time. Missing tables are treated as already clean. + // + // Both families are swept, not just the inet one this now installs into: a + // table name is unique per family, so an ip table left by an earlier ateom + // would survive every later cleanup and keep redirecting alongside the new + // one. c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("while listing nftables tables: %w", err) - } - for _, table := range tables { - if table.Name != ActorNftTableName { - continue + for _, family := range []nftables.TableFamily{nftables.TableFamilyINet, nftables.TableFamilyIPv4} { + tables, err := c.ListTablesOfFamily(family) + if err != nil { + return fmt.Errorf("while listing nftables tables: %w", err) } - c.DelTable(table) - if err := c.Flush(); err != nil { - return fmt.Errorf("while deleting actor nftables table: %w", err) + for _, table := range tables { + if table.Name != ActorNftTableName { + continue + } + c.DelTable(table) + if err := c.Flush(); err != nil { + return fmt.Errorf("while deleting actor nftables table: %w", err) + } } - return nil } return nil } @@ -329,6 +470,12 @@ func IPSourceEqual(ip string) []expr.Any { func IPPayloadEqual(offset uint32, ip string) []expr.Any { return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.NFPROTO_IPV4}, + }, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, @@ -343,6 +490,32 @@ func IPPayloadEqual(offset uint32, ip string) []expr.Any { } } +func IPv6SourceEqual(ip string) []expr.Any { + return IPv6PayloadEqual(8, ip) +} + +func IPv6PayloadEqual(offset uint32, ip string) []expr.Any { + return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.NFPROTO_IPV6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: 16, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: MustParseIPv6(ip), + }, + } +} + func TCPProtocol() []expr.Any { return []expr.Any{ &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, @@ -372,6 +545,24 @@ func ActorEgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} } +// ActorIPv6EgressRedirectRule is ActorEgressRedirectRule for the actor's IPv6 +// source address. Both rules live in the same inet table, so each carries its +// own NFPROTO match to keep it off the other family's packets. +func ActorIPv6EgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port uint16) *nftables.Rule { + if port == 0 { + return nil + } + exprs := append(IPv6SourceEqual(ActorVethIPv6IP), TCPProtocol()...) + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(port), + }, + &expr.Redir{RegisterProtoMin: 1}, + ) + return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} +} + // CreateNetNSWithoutSwitching creates a named netns and returns its handle, // restoring the caller's current netns before returning. func CreateNetNSWithoutSwitching(name string) (netns.NsHandle, error) { @@ -557,15 +748,35 @@ func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { if err := netlink.AddrReplace(hostLink, HostVethAddr); err != nil { return fmt.Errorf("while assigning host veth address: %w", err) } + // Decided once, here in the worker pod netns, and carried into the interior + // netns below. Probing separately on each side would let them disagree: the + // interior netns is freshly created, so its sysctl is always the permissive + // kernel default whatever the pod's families are. + var podIPv6 bool + if podLink, err := netlink.LinkByName(podPrimaryIfaceName); err == nil { + podIPv6 = linkHasGlobalIPv6(ctx, podLink) + } + vethIPv6 := linkIPv6Enabled(HostVethName) + actorIPv6 := podIPv6 && vethIPv6 + if actorIPv6 { + if err := netlink.AddrReplace(hostLink, HostVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning host veth ipv6 address: %w", err) + } + } else { + slog.InfoContext(ctx, "actor networking is IPv4-only", + "link", HostVethName, "podHasGlobalIPv6", podIPv6, "vethIPv6Enabled", vethIPv6) + } if err := netlink.LinkSetUp(hostLink); err != nil { return fmt.Errorf("while bringing up host veth: %w", err) } - if err := NetNSDo(ctx, cfg.InteriorNetNS, ConfigureActorVeth); err != nil { + if err := NetNSDo(ctx, cfg.InteriorNetNS, func(ctx context.Context) error { + return ConfigureActorVeth(ctx, actorIPv6) + }); err != nil { return fmt.Errorf("while configuring actor veth in interior netns: %w", err) } - if err := EnableIPv4Forwarding(); err != nil { + if err := EnableForwarding(); err != nil { return err } if err := InstallActorNftablesRules(cfg.EgressRedirectPort); err != nil { diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index b9c8ac45f4..bbaddc78ec 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -19,6 +19,8 @@ package ateomnet import ( "context" "errors" + "net" + "os" "runtime" "testing" @@ -26,6 +28,7 @@ import ( "github.com/google/nftables" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" + "golang.org/x/sys/unix" ) // withTestNetNS runs fn with the calling thread inside a throwaway netns @@ -86,6 +89,26 @@ func requireNftables(t *testing.T) { } } +// actorNftTableExists reports whether the actor table is present in the family +// InstallActorNftablesRules creates it in. The family is load-bearing: +// ListTablesOfFamily puts it in the netlink dump header, so the kernel filters +// the dump and a query for the wrong family comes back empty rather than +// erroring. +func actorNftTableExists(t *testing.T) bool { + t.Helper() + c := &nftables.Conn{} + tables, err := c.ListTablesOfFamily(nftables.TableFamilyINet) + if err != nil { + t.Fatalf("listing inet nftables tables: %v", err) + } + for _, table := range tables { + if table.Name == ActorNftTableName { + return true + } + } + return false +} + // linkByName returns the link, or nil when it does not exist. func linkByName(t *testing.T, name string) netlink.Link { t.Helper() @@ -115,6 +138,39 @@ func hasAddr(t *testing.T, link netlink.Link, cidr string) bool { return false } +// assertDefaultRoute requires link to carry -- or, when want is false, to not +// carry -- a default route via gw in the given family. +func assertDefaultRoute(t *testing.T, link netlink.Link, family int, gw net.IP, want bool) { + t.Helper() + + dst := "0.0.0.0/0" + if family == netlink.FAMILY_V6 { + dst = "::/0" + } + routes, err := netlink.RouteList(link, family) + if err != nil { + t.Fatalf("listing %s routes of %q: %v", dst, link.Attrs().Name, err) + } + var got bool + for _, route := range routes { + // A default route reports its destination either as nil or as an + // explicit zero-length mask, depending on how the kernel rendered it. + ones := 0 + if route.Dst != nil { + ones, _ = route.Dst.Mask.Size() + } + if ones == 0 && route.Gw.Equal(gw) { + got = true + } + } + switch { + case want && !got: + t.Errorf("%q has no %s route via %s, got %v", link.Attrs().Name, dst, gw, routes) + case !want && got: + t.Errorf("%q has a %s route via %s, want none, got %v", link.Attrs().Name, dst, gw, routes) + } +} + // TestSetupActorNetworkFinalState pins the namespace state gVisor and the // micro-VM guest read after an activation: what links exist, where, with which // addresses and routes. It deliberately asserts the end state rather than the @@ -169,28 +225,7 @@ func TestSetupActorNetworkFinalState(t *testing.T) { t.Error("interior loopback is not up") } - routes, err := netlink.RouteList(actor, netlink.FAMILY_V4) - if err != nil { - t.Fatalf("listing interior routes: %v", err) - } - // A default route reports its destination either as nil or as an - // explicit 0.0.0.0/0, depending on how the kernel rendered it. - isDefault := func(route netlink.Route) bool { - if route.Dst == nil { - return true - } - ones, _ := route.Dst.Mask.Size() - return ones == 0 - } - var haveDefault bool - for _, route := range routes { - if isDefault(route) && route.Gw.Equal(ActorVethGwIP) { - haveDefault = true - } - } - if !haveDefault { - t.Errorf("interior netns has no default route via %s, got %v", ActorVethGateway, routes) - } + assertDefaultRoute(t, actor, netlink.FAMILY_V4, ActorVethGwIP, true) return nil }); err != nil { t.Fatalf("inspecting interior netns: %v", err) @@ -215,9 +250,20 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { if linkByName(t, HostVethName) == nil { t.Fatalf("host veth %q missing after activation %d", HostVethName, i) } + if !actorNftTableExists(t) { + t.Fatalf("nftables table %q missing after activation %d", ActorNftTableName, i) + } if err := CleanupActorNetwork(ctx, interior); err != nil { t.Fatalf("CleanupActorNetwork (activation %d): %v", i, err) } + // Install and teardown have to name the same family. When they do not, + // teardown's dump comes back empty, its "missing tables are already + // clean" path reports success, and the table survives -- so the next + // activation stacks another copy of every chain and rule onto it and + // the leak is invisible to every other assertion here. + if actorNftTableExists(t) { + t.Fatalf("nftables table %q survived cleanup after activation %d", ActorNftTableName, i) + } } // Cleanup is idempotent: the extra call after the loop's last one must @@ -228,6 +274,9 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { if stray := linkByName(t, HostVethName); stray != nil { t.Errorf("host veth %q survived cleanup", HostVethName) } + if actorNftTableExists(t) { + t.Errorf("nftables table %q survived a repeated cleanup", ActorNftTableName) + } if err := NetNSDo(ctx, interior, func(context.Context) error { if stray := linkByName(t, ActorVethName); stray != nil { t.Errorf("actor veth %q survived cleanup", ActorVethName) @@ -239,6 +288,223 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { }) } +// TestRemoveActorNftablesRulesSweepsIPv4Family covers the upgrade case: a +// worker whose previous ateom created the actor table in the ip family. Table +// names are unique per family, so an inet-only cleanup could never see that +// table, and it would have kept redirecting alongside the inet one installed +// next to it. +func TestRemoveActorNftablesRulesSweepsIPv4Family(t *testing.T) { + roottest.Require(t, "creating network namespaces and nftables rules") + + withTestNetNS(t, func(netns.NsHandle) { + requireNftables(t) + + c := &nftables.Conn{} + c.AddTable(&nftables.Table{Family: nftables.TableFamilyIPv4, Name: ActorNftTableName}) + if err := c.Flush(); err != nil { + t.Fatalf("creating the stand-in ip actor table: %v", err) + } + + if err := RemoveActorNftablesRules(); err != nil { + t.Fatalf("RemoveActorNftablesRules: %v", err) + } + + tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) + if err != nil { + t.Fatalf("listing ip nftables tables: %v", err) + } + for _, table := range tables { + if table.Name == ActorNftTableName { + t.Fatal("the ip actor table survived cleanup") + } + } + }) +} + +// addPodEth0 plants a dummy link carrying cidrs in the current netns, standing +// in for the worker pod's own primary interface. withTestNetNS hands out a bare +// namespace, and the families on that interface are what SetupActorNetwork reads +// to decide the families the actor gets. +// +// The name has to be exactly podPrimaryIfaceName: the probe is link-scoped, so +// under any other name it answers false and the test asserts the opposite of +// what it means to. +func addPodEth0(t *testing.T, cidrs ...string) { + t.Helper() + + link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: podPrimaryIfaceName}} + if err := netlink.LinkAdd(link); err != nil { + t.Fatalf("creating the stand-in pod %s: %v", podPrimaryIfaceName, err) + } + if err := netlink.LinkSetUp(link); err != nil { + t.Fatalf("bringing up the stand-in pod %s: %v", podPrimaryIfaceName, err) + } + for _, cidr := range cidrs { + addr := MustParseAddr(cidr) + addr.Flags |= unix.IFA_F_NODAD // else an IPv6 address stays tentative + if err := netlink.AddrAdd(link, addr); err != nil { + t.Fatalf("assigning %s to the stand-in pod %s: %v", cidr, podPrimaryIfaceName, err) + } + } +} + +// writeSysctl turns an IPv6 knob off in the current netns. "all" flushes the +// addresses already assigned; "default" only reaches links created afterwards. +func writeSysctl(t *testing.T, knob string) { + t.Helper() + path := "/proc/sys/net/ipv6/conf/" + knob + "/disable_ipv6" + if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { + t.Fatalf("disabling IPv6 via %s: %v", path, err) + } +} + +// assertIPv6AddrNoDAD requires cidr to be present on link and to carry +// IFA_F_NODAD. +// +// The flag is the whole point: the ateom container is unprivileged, so the +// accept_dad sysctl this replaced could not be written and setup failed outright +// on a real worker. It passes as root, where /proc/sys is writable either way, +// so nothing else here would catch a regression back to the sysctl. +func assertIPv6AddrNoDAD(t *testing.T, link netlink.Link, cidr string) { + t.Helper() + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + t.Fatalf("listing IPv6 addresses of %q: %v", link.Attrs().Name, err) + } + want := MustParseAddr(cidr) + for _, addr := range addrs { + if addr.IPNet == nil || addr.IPNet.String() != want.IPNet.String() { + continue + } + if addr.Flags&unix.IFA_F_NODAD == 0 { + t.Errorf("%s on %q has flags %#x, want IFA_F_NODAD (%#x) set", cidr, link.Attrs().Name, addr.Flags, unix.IFA_F_NODAD) + } + return + } + t.Errorf("%q does not carry %s, got %v", link.Attrs().Name, cidr, addrs) +} + +// assertNoGlobalIPv6Addr requires link to carry no IPv6 address beyond the +// fe80::/64 the kernel gives every up link wherever IPv6 is enabled at all. +// That link-local is not what strands an actor -- the routable address is. +func assertNoGlobalIPv6Addr(t *testing.T, link netlink.Link) { + t.Helper() + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + t.Fatalf("listing IPv6 addresses of %q: %v", link.Attrs().Name, err) + } + for _, addr := range addrs { + if addr.IP.IsGlobalUnicast() { + t.Errorf("%q carries global IPv6 address %s, want none", link.Attrs().Name, addr) + } + } +} + +// TestSetupActorNetworkIPv6Gate is the truth table for who gets actor IPv6. +// Both halves have to hold: the worker pod needs a global IPv6 address of its +// own, or the actor prefers the AAAA of a dual-stack destination and the +// connection dies with nowhere to go; and the veth has to accept an IPv6 +// address at all, or the assignment fails with EPERM on the path of every +// SetupActorNetwork call and the actor never starts. +// +// The IPv4 half must come out identical in every case. +func TestSetupActorNetworkIPv6Gate(t *testing.T) { + roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") + ctx := context.Background() + + for _, tc := range []struct { + name string + // podAddrs go on the stand-in pod interface before setup runs. + podAddrs []string + // disable, when set, runs in the pod netns after podAddrs are assigned. + disable func(*testing.T) + wantIPv6 bool + }{{ + name: "dual-stack pod", + podAddrs: []string{"10.244.0.7/24", "fd00:10:244::7/64"}, + wantIPv6: true, + }, { + // A probe that reads the wrong link or the wrong scope fails closed, + // which every IPv4 case here would happily accept. This one notices. + name: "IPv6-only pod", + podAddrs: []string{"fd00:10:244::7/64"}, + wantIPv6: true, + }, { + // An IPv4-only cluster whose kernel still has IPv6 compiled in, so every + // capability probe says yes. This is the case that turned the IPv4 e2e + // job red. + name: "pod without IPv6", + podAddrs: []string{"10.244.0.7/24"}, + }, { + // The default on IPv4-only GKE. Writing "all" also flushes podAddrs, so + // both halves of the gate are false here. + name: "IPv6 disabled for the whole netns", + podAddrs: []string{"10.244.0.7/24", "fd00:10:244::7/64"}, + disable: func(t *testing.T) { + writeSysctl(t, "all") + writeSysctl(t, "default") + }, + }, { + // The one case the capability half is there for: the pod keeps its + // address, but the veth created next inherits disable_ipv6=1. + name: "IPv6 disabled per link", + podAddrs: []string{"10.244.0.7/24", "fd00:10:244::7/64"}, + disable: func(t *testing.T) { writeSysctl(t, "default") }, + }} { + t.Run(tc.name, func(t *testing.T) { + withTestNetNS(t, func(interior netns.NsHandle) { + requireNftables(t) + + addPodEth0(t, tc.podAddrs...) + if tc.disable != nil { + tc.disable(t) + } + + if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { + t.Fatalf("SetupActorNetwork: %v", err) + } + + host := linkByName(t, HostVethName) + if host == nil { + t.Fatalf("host veth %q missing from the pod netns", HostVethName) + } + if !hasAddr(t, host, HostVethCIDR) { + t.Errorf("host veth %q does not carry %s", HostVethName, HostVethCIDR) + } + if tc.wantIPv6 { + assertIPv6AddrNoDAD(t, host, HostVethIPv6CIDR) + } else { + assertNoGlobalIPv6Addr(t, host) + } + + if err := NetNSDo(ctx, interior, func(context.Context) error { + actor := linkByName(t, ActorVethName) + if actor == nil { + t.Fatalf("actor veth %q missing from the interior netns", ActorVethName) + } + if !hasAddr(t, actor, ActorVethCIDR) { + t.Errorf("actor veth %q does not carry %s", ActorVethName, ActorVethCIDR) + } + assertDefaultRoute(t, actor, netlink.FAMILY_V4, ActorVethGwIP, true) + + // The interior netns is created fresh, so its own sysctls always + // say IPv6 is available whatever the pod's families are. Only a + // decision carried across from the pod netns gets this right. + if tc.wantIPv6 { + assertIPv6AddrNoDAD(t, actor, ActorVethIPv6CIDR) + } else { + assertNoGlobalIPv6Addr(t, actor) + } + assertDefaultRoute(t, actor, netlink.FAMILY_V6, ActorVethIPv6GwIP, tc.wantIPv6) + return nil + }); err != nil { + t.Fatalf("inspecting interior netns: %v", err) + } + }) + }) + } +} + // TestSetupActorNetworkHostVethHWAddr covers the micro-VM requirement: a CH // snapshot freezes the guest's ARP entry for the gateway, so the worker-side // veth MAC has to be exactly the one the caller asked for, on every pod. diff --git a/internal/ateomnet/write_sysctl_test.go b/internal/ateomnet/write_sysctl_test.go new file mode 100644 index 0000000000..d687932d8c --- /dev/null +++ b/internal/ateomnet/write_sysctl_test.go @@ -0,0 +1,99 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "os" + "path/filepath" + "testing" +) + +// TestWriteSysctlIfUnset verifies writeSysctlIfUnset's fast paths against a +// temp file standing in for a /proc/sys node: it must not rewrite a value +// that already reads "1", and it must write "1\n" when the value is missing +// or unset. The privileged bind-remount path is covered by the netns +// integration tests (withTestNetNS), which require root. +func TestWriteSysctlIfUnset(t *testing.T) { + dir := t.TempDir() + + t.Run("already_set", func(t *testing.T) { + p := filepath.Join(dir, "already") + // Sentinel content: if writeSysctlIfUnset rewrote the file, the value + // would change to "1\n" and this assertion would fail. Keeping the + // file larger than the helper's output makes a silent rewrite + // detectable. + if err := os.WriteFile(p, []byte("1 other-content\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if string(b) != "1 other-content\n" { + t.Fatalf("already-set file was rewritten: %q", b) + } + }) + + t.Run("unset_written", func(t *testing.T) { + p := filepath.Join(dir, "unset") + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if len(b) < 1 || b[0] != '1' { + t.Fatalf("expected '1' written, got %q", b) + } + }) + + t.Run("missing_path_is_noop", func(t *testing.T) { + // A node under a directory that does not exist stands in for + // /proc/sys/net/ipv6/... on a kernel with IPv6 disabled. The other + // subtests' paths can be created, so they return at the os.WriteFile + // fast path; this is the only one that reaches the os.Stat branch, + // which is what procfs always does in production. + p := filepath.Join(dir, "no-such-dir", "forwarding") + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset on a missing path: %v", err) + } + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Fatalf("expected %s to stay absent, stat err = %v", p, err) + } + }) + + t.Run("zero_is_rewritten", func(t *testing.T) { + p := filepath.Join(dir, "zero") + if err := os.WriteFile(p, []byte("0\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if len(b) < 1 || b[0] != '1' { + t.Fatalf("expected '1' written, got %q", b) + } + }) +} diff --git a/internal/atunnel/original_dst_format_linux_test.go b/internal/atunnel/original_dst_format_linux_test.go new file mode 100644 index 0000000000..0558af1834 --- /dev/null +++ b/internal/atunnel/original_dst_format_linux_test.go @@ -0,0 +1,107 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "encoding/binary" + "net" + "strings" + "testing" +) + +// networkOrderPort produces the raw field value the kernel leaves in +// RawSockaddrInet4.Port and RawSockaddrInet6.Port: a uint16 whose in-memory +// bytes are the port in network order, which on a little-endian host is not +// the port's numeric value. +func networkOrderPort(port uint16) uint16 { + return binary.NativeEndian.Uint16(binary.BigEndian.AppendUint16(nil, port)) +} + +func TestFormatOriginalDestination(t *testing.T) { + tests := []struct { + name string + ip []byte + port uint16 + want string + wantErr bool + }{ + { + name: "IPv4", + ip: []byte{198, 18, 0, 1}, + port: 443, + want: "198.18.0.1:443", + }, + { + name: "IPv6 is bracketed", + ip: net.ParseIP("fd00:198:18::1").To16(), + port: 443, + // SplitHostPort in the atunnel client needs the brackets. + want: "[fd00:198:18::1]:443", + }, + { + name: "v4-mapped IPv6 renders as IPv4", + ip: net.ParseIP("::ffff:198.18.0.1").To16(), + port: 8080, + want: "198.18.0.1:8080", + }, + { + name: "high port is not sign-extended", + ip: []byte{198, 18, 0, 1}, + port: 65535, + want: "198.18.0.1:65535", + }, + { + // A zero port means the lookup answered without a real destination, + // which would otherwise become a dial to port 0. + name: "port zero is rejected", + ip: []byte{198, 18, 0, 1}, + port: 0, + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := formatOriginalDestination(test.ip, networkOrderPort(test.port)) + if test.wantErr { + if err == nil { + t.Fatalf("formatOriginalDestination() = %q, want an error", got) + } + return + } + if err != nil { + t.Fatalf("formatOriginalDestination() error = %v", err) + } + if got != test.want { + t.Errorf("formatOriginalDestination() = %q, want %q", got, test.want) + } + }) + } +} + +func TestTCPOriginalDestinationRejectsNonTCPConn(t *testing.T) { + client, server := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + t.Cleanup(func() { _ = server.Close() }) + + got, err := TCPOriginalDestination(client) + if err == nil { + t.Fatalf("TCPOriginalDestination() = %q, want an error on a non-TCP connection", got) + } + if !strings.Contains(err.Error(), "requires a TCP connection") { + t.Errorf("TCPOriginalDestination() error = %v, want it to name the unsupported connection type", err) + } +} diff --git a/internal/atunnel/original_dst_linux.go b/internal/atunnel/original_dst_linux.go index 07dd0f9344..4a5cb33b3f 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/atunnel/original_dst_linux.go @@ -26,10 +26,14 @@ import ( "golang.org/x/sys/unix" ) -// TCPOriginalDestination reads the IPv4 destination preserved by a Linux -// REDIRECT rule. Actor networking is currently IPv4-only. -// TODO(liorlieberman) add the IPv6 IP6T_SO_ORIGINAL_DST variant -// when actor veth setup gains dual-stack support. +// IP6T_SO_ORIGINAL_DST is not generated by golang.org/x/sys/unix. It is defined +// as 80 in linux/netfilter_ipv6/ip6_tables.h — the same number as +// unix.SO_ORIGINAL_DST by coincidence, not by definition, since the two are +// options of different levels. +const ip6tSOOriginalDst = 80 + +// TCPOriginalDestination reads the IPv4 or IPv6 destination preserved by a +// Linux REDIRECT rule. func TCPOriginalDestination(conn net.Conn) (string, error) { tcpConn, ok := conn.(*net.TCPConn) if !ok { @@ -39,34 +43,72 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { if err != nil { return "", fmt.Errorf("atunnel: acquiring TCP syscall connection: %w", err) } + // Each family keeps its original destination under its own socket option + // level, and querying the other one answers EOPNOTSUPP rather than anything + // about the flow. A v4-mapped local address still means an IPv4 flow, so To4 + // is the test, not the socket domain. + local, ok := tcpConn.LocalAddr().(*net.TCPAddr) + if !ok { + return "", fmt.Errorf("atunnel: original destination requires a TCP local address, got %T", tcpConn.LocalAddr()) + } + isIPv6 := local.IP.To4() == nil - var addr unix.RawSockaddrInet4 var sockoptErr error + var destination string if err := rawConn.Control(func(fd uintptr) { - size := uint32(unsafe.Sizeof(addr)) - _, _, errno := unix.Syscall6( - unix.SYS_GETSOCKOPT, - fd, - unix.SOL_IP, - unix.SO_ORIGINAL_DST, - uintptr(unsafe.Pointer(&addr)), - uintptr(unsafe.Pointer(&size)), - 0, - ) - if errno != 0 { - sockoptErr = errno + if isIPv6 { + destination, sockoptErr = originalIPv6Destination(fd) + return } + destination, sockoptErr = originalIPv4Destination(fd) }); err != nil { return "", fmt.Errorf("atunnel: accessing TCP socket: %w", err) } if sockoptErr != nil { - return "", fmt.Errorf("atunnel: reading original TCP destination: %w", sockoptErr) + family := "IPv4" + if isIPv6 { + family = "IPv6" + } + return "", fmt.Errorf("atunnel: reading original %s TCP destination: %w", family, sockoptErr) } + return destination, nil +} + +func originalIPv4Destination(fd uintptr) (string, error) { + var addr unix.RawSockaddrInet4 + if errno := getOriginalDestination(fd, unix.SOL_IP, unix.SO_ORIGINAL_DST, unsafe.Pointer(&addr), unsafe.Sizeof(addr)); errno != 0 { + return "", errno + } + return formatOriginalDestination(addr.Addr[:], addr.Port) +} + +func originalIPv6Destination(fd uintptr) (string, error) { + var addr unix.RawSockaddrInet6 + if errno := getOriginalDestination(fd, unix.SOL_IPV6, ip6tSOOriginalDst, unsafe.Pointer(&addr), unsafe.Sizeof(addr)); errno != 0 { + return "", errno + } + return formatOriginalDestination(addr.Addr[:], addr.Port) +} + +func getOriginalDestination(fd uintptr, level, option int, addr unsafe.Pointer, addrSize uintptr) unix.Errno { + size := uint32(addrSize) + _, _, errno := unix.Syscall6( + unix.SYS_GETSOCKOPT, + fd, + uintptr(level), + uintptr(option), + uintptr(addr), + uintptr(unsafe.Pointer(&size)), + 0, + ) + return errno +} - portBytes := (*[2]byte)(unsafe.Pointer(&addr.Port)) +func formatOriginalDestination(ip []byte, rawPort uint16) (string, error) { + portBytes := (*[2]byte)(unsafe.Pointer(&rawPort)) port := binary.BigEndian.Uint16(portBytes[:]) if port == 0 { return "", fmt.Errorf("atunnel: original TCP destination has port zero") } - return net.JoinHostPort(net.IP(addr.Addr[:]).String(), strconv.Itoa(int(port))), nil + return net.JoinHostPort(net.IP(ip).String(), strconv.Itoa(int(port))), nil } diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go new file mode 100644 index 0000000000..3e0ae61d1f --- /dev/null +++ b/internal/atunnel/original_dst_linux_test.go @@ -0,0 +1,409 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "context" + "errors" + "fmt" + "net" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/roottest" +) + +// originalDstFamily parameterizes the redirect test. The wiring is the same in +// every case; only the addresses, the nftables table family, the source-address +// matcher and the sockets on either end differ. +type originalDstFamily struct { + name string + // listenNetwork and listenIP describe the worker's listeners; an empty + // listenIP binds the unspecified address. dialNetwork is what the actor + // dials with, which is not always the same family the listener was opened + // as. + listenNetwork string + listenIP string + dialNetwork string + nftFamily nftables.TableFamily + workerIP string + actorIP string + mask net.IPMask + addrFlags int + sourceEqual func(string) []expr.Any +} + +// Fixed addresses are safe because each test builds its own namespaces. +var originalDstFamilies = []originalDstFamily{ + { + name: "IPv4", + listenNetwork: "tcp4", + listenIP: "198.18.0.1", + dialNetwork: "tcp4", + nftFamily: nftables.TableFamilyIPv4, + workerIP: "198.18.0.1", + actorIP: "198.18.0.2", + mask: net.CIDRMask(30, 32), + sourceEqual: ateomnet.IPSourceEqual, + }, + { + name: "IPv6", + listenNetwork: "tcp6", + listenIP: "fd00:198:18::1", + dialNetwork: "tcp6", + nftFamily: nftables.TableFamilyIPv6, + workerIP: "fd00:198:18::1", + actorIP: "fd00:198:18::2", + mask: net.CIDRMask(64, 128), + // The veth is alone in a throwaway namespace, so nothing can collide + // with it. Skipping DAD lets the listener bind straight away instead of + // waiting out the tentative period. + addrFlags: unix.IFA_F_NODAD, + sourceEqual: ateomnet.IPv6SourceEqual, + }, + { + // atunnel listens on an unspecified address, which Go opens as one + // dual-stack AF_INET6 socket, so an IPv4 actor arrives there with a + // v4-mapped local address and its original destination is still only + // readable through the IPv4 socket option. This is the shape production + // runs in and the one the family check exists for. + name: "DualStackV4Mapped", + listenNetwork: "tcp", + dialNetwork: "tcp4", + nftFamily: nftables.TableFamilyIPv4, + workerIP: "198.18.0.1", + actorIP: "198.18.0.2", + mask: net.CIDRMask(30, 32), + sourceEqual: ateomnet.IPSourceEqual, + }, +} + +// TestTCPOriginalDestinationRedirect models the production egress path: actor +// traffic enters the worker namespace over a veth, an nftables PREROUTING rule +// redirects it to a local listener, and that listener asks the kernel what the +// actor originally dialed. Redirecting a locally generated connection through +// OUTPUT would not exercise the same path. +// +// The worker side gets its own namespace rather than borrowing the host's. That +// keeps the test clear of any local firewall policy — a default-deny INPUT +// chain would otherwise drop the redirected SYN — and the veth and nftables +// table go away with the namespace instead of needing to be swept up. +func TestTCPOriginalDestinationRedirect(t *testing.T) { + roottest.Require(t, "CAP_NET_ADMIN + CAP_SYS_ADMIN for network namespaces and an nftables REDIRECT rule") + + for _, family := range originalDstFamilies { + t.Run(family.name, func(t *testing.T) { + workerNS := newTestNetNS(t) + actorNS := newTestNetNS(t) + requireNftables(t, workerNS) + setupTestVeth(t, family, workerNS, actorNS) + + // targetListener holds the port the actor means to reach, so the + // assertion below cannot pass by the connection simply arriving + // where it was aimed. redirectListener stands in for atunnel's own + // egress listener and is where the redirect must land instead. + redirectListener := listenInNetNS(t, workerNS, family) + targetListener := listenInNetNS(t, workerNS, family) + targetPort := targetListener.Addr().(*net.TCPAddr).Port + redirectPort := redirectListener.Addr().(*net.TCPAddr).Port + installOriginalDstRedirect(t, family, workerNS, targetPort, redirectPort) + + target := net.JoinHostPort(family.workerIP, strconv.Itoa(targetPort)) + clientDone := make(chan error, 1) + go func() { + // From the actor's side this is an ordinary connection to the + // worker's address; PREROUTING rewrites it on the way in. + clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + conn, err := net.DialTimeout(family.dialNetwork, target, 10*time.Second) + if err != nil { + return err + } + return conn.Close() + }) + }() + + if err := redirectListener.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatalf("setting the accept deadline: %v", err) + } + redirected, err := redirectListener.Accept() + if err != nil { + t.Fatalf("accepting the redirected connection: %v", err) + } + defer redirected.Close() + + // The accepted socket is addressed to redirectListener; the kernel's + // conntrack record must still hold what the actor dialed. + got, err := TCPOriginalDestination(redirected) + if err != nil { + t.Fatalf("TCPOriginalDestination: %v", err) + } + if got != target { + t.Errorf("original destination = %q, want %q", got, target) + } + if err := <-clientDone; err != nil { + t.Fatalf("dialing through the redirect: %v", err) + } + }) + } +} + +// TestTCPOriginalDestinationPreservesErrno covers the failure path on an +// ordinary connection that no REDIRECT rule touched. Each family's lookup +// misses and reports ENOENT, and that error must reach the caller rather than +// the EOPNOTSUPP a single-family socket answers for the other family's option. +// A dual-stack socket answers both with ENOENT, so the errno alone does not +// say which lookup ran; the family named in the message does. +// +// The dual-stack case is the shape production actually sees. atunnel listens +// on an unspecified address, which Go opens as an AF_INET6 socket with +// IPV6_V6ONLY off, so every IPv4 actor connection arrives with a v4-mapped +// local address and must still take the IPv4 lookup. +// +// These run in a fresh namespace because conntrack tracks loopback in any +// namespace that has nftables rules — including the one Docker runs in — and a +// tracked connection returns its real destination instead of missing. +func TestTCPOriginalDestinationPreservesErrno(t *testing.T) { + roottest.Require(t, "CAP_SYS_ADMIN for a network namespace with no conntrack hooks") + + tests := []struct { + name string + listenNetwork string + listenAddress string + dialNetwork string + dialHost string + // wantFamily is the family the error has to name. Both lookups miss + // with the same errno, so this is the only thing that distinguishes + // the one that ran from the one that should have. + wantFamily string + }{ + {name: "IPv4", listenNetwork: "tcp4", listenAddress: "127.0.0.1:0", dialNetwork: "tcp4", dialHost: "127.0.0.1", wantFamily: "IPv4"}, + {name: "IPv6", listenNetwork: "tcp6", listenAddress: "[::1]:0", dialNetwork: "tcp6", dialHost: "::1", wantFamily: "IPv6"}, + {name: "DualStackV4Mapped", listenNetwork: "tcp", listenAddress: ":0", dialNetwork: "tcp4", dialHost: "127.0.0.1", wantFamily: "IPv4"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ns := newTestNetNS(t) + // Recorded rather than asserted in place: t.Skipf unwinds the + // goroutine, and NetNSDo has the thread switched into another + // namespace at that point. + var lookupErr error + if err := ateomnet.NetNSDo(context.Background(), ns, func(context.Context) error { + listener, err := net.Listen(test.listenNetwork, test.listenAddress) + if err != nil { + return err + } + defer listener.Close() + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + return err + } + client, err := net.DialTimeout(test.dialNetwork, net.JoinHostPort(test.dialHost, port), 10*time.Second) + if err != nil { + return err + } + defer client.Close() + server, err := listener.Accept() + if err != nil { + return err + } + defer server.Close() + + _, lookupErr = TCPOriginalDestination(server) + return nil + }); err != nil { + t.Fatal(err) + } + if errors.Is(lookupErr, unix.ENOPROTOOPT) { + // A kernel built without the conntrack socket option handler + // cannot answer either family, so there is nothing to assert. + t.Skipf("the kernel does not serve SO_ORIGINAL_DST: %v", lookupErr) + } + if !errors.Is(lookupErr, unix.ENOENT) { + t.Errorf("want a lookup miss reported as ENOENT, got %v", lookupErr) + } + if want := "original " + test.wantFamily + " TCP destination"; !strings.Contains(lookupErr.Error(), want) { + t.Errorf("want the error to report %q, got %v", want, lookupErr) + } + }) + } +} + +// newTestNetNS returns a throwaway network namespace with its loopback up. It +// is anonymous rather than named: there is no /var/run/netns bind mount to +// collide with a concurrent run or to leak if the process is killed, and +// closing the handle takes every link and nftables table in it away. +func newTestNetNS(t *testing.T) netns.NsHandle { + t.Helper() + // A namespace is a property of the thread, and netns.New switches the + // caller into the one it creates, so the thread has to be pinned until we + // have switched back. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + current, err := netns.Get() + if err != nil { + t.Fatalf("getting the current netns: %v", err) + } + defer current.Close() + // Registered before the namespace below so it runs after it is created. + defer func() { + if err := netns.Set(current); err != nil { + t.Errorf("restoring the original netns: %v", err) + } + }() + + ns, err := netns.New() + if err != nil { + if errors.Is(err, unix.EPERM) { + t.Skipf("needs CAP_SYS_ADMIN to create a network namespace: %v", err) + } + t.Fatalf("creating a test netns: %v", err) + } + t.Cleanup(func() { _ = ns.Close() }) + + loopback, err := netlink.LinkByName("lo") + if err != nil { + t.Fatalf("looking up lo in the test netns: %v", err) + } + if err := netlink.LinkSetUp(loopback); err != nil { + t.Fatalf("bringing lo up in the test netns: %v", err) + } + return ns +} + +// requireNftables skips when the kernel in this environment cannot serve the +// nftables netlink API at all, which is a property of the machine rather than +// of the code under test. +func requireNftables(t *testing.T, ns netns.NsHandle) { + t.Helper() + c, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err == nil { + _, err = c.ListTablesOfFamily(nftables.TableFamilyIPv4) + } + if err != nil { + t.Skipf("nftables unavailable in this environment: %v", err) + } +} + +// setupTestVeth joins the two namespaces with an addressed point-to-point veth. +func setupTestVeth(t *testing.T, family originalDstFamily, workerNS, actorNS netns.NsHandle) { + t.Helper() + const workerEnd, actorEnd = "atodw", "atoda" + if err := ateomnet.NetNSDo(context.Background(), workerNS, func(context.Context) error { + if err := netlink.LinkAdd(&netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{Name: workerEnd}, + PeerName: actorEnd, + }); err != nil { + return fmt.Errorf("creating the veth: %w", err) + } + peer, err := netlink.LinkByName(actorEnd) + if err != nil { + return err + } + if err := netlink.LinkSetNsFd(peer, int(actorNS)); err != nil { + return fmt.Errorf("moving the actor end into its namespace: %w", err) + } + return configureVethEnd(family, workerEnd, family.workerIP) + }); err != nil { + if errors.Is(err, unix.EPERM) { + t.Skipf("needs CAP_NET_ADMIN to create a veth: %v", err) + } + t.Fatalf("wiring the worker end of the veth: %v", err) + } + if err := ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + return configureVethEnd(family, actorEnd, family.actorIP) + }); err != nil { + t.Fatalf("wiring the actor end of the veth: %v", err) + } +} + +func configureVethEnd(family originalDstFamily, name, ip string) error { + link, err := netlink.LinkByName(name) + if err != nil { + return err + } + addr := &netlink.Addr{ + IPNet: &net.IPNet{IP: net.ParseIP(ip), Mask: family.mask}, + Flags: family.addrFlags, + } + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("adding %s to %s: %w", ip, name, err) + } + return netlink.LinkSetUp(link) +} + +// listenInNetNS opens a listener inside ns. The socket stays bound to that +// namespace once created, so the caller can accept on it from wherever it +// happens to be running. +func listenInNetNS(t *testing.T, ns netns.NsHandle, family originalDstFamily) *net.TCPListener { + t.Helper() + var listener *net.TCPListener + if err := ateomnet.NetNSDo(context.Background(), ns, func(context.Context) error { + l, err := net.ListenTCP(family.listenNetwork, &net.TCPAddr{IP: net.ParseIP(family.listenIP)}) + if err != nil { + return err + } + listener = l + return nil + }); err != nil { + t.Fatalf("listening on %s %q: %v", family.listenNetwork, family.listenIP, err) + } + t.Cleanup(func() { _ = listener.Close() }) + return listener +} + +// installOriginalDstRedirect sends the actor's connections to targetPort on to +// redirectPort instead, the way a worker sends actor egress to atunnel. +func installOriginalDstRedirect(t *testing.T, family originalDstFamily, ns netns.NsHandle, targetPort, redirectPort int) { + t.Helper() + c, err := nftables.New(nftables.WithNetNSFd(int(ns))) + if err != nil { + t.Fatalf("opening nftables in the worker namespace: %v", err) + } + table := c.AddTable(&nftables.Table{Family: family.nftFamily, Name: "atunnel_original_dst_test"}) + chain := c.AddChain(&nftables.Chain{ + Name: "prerouting", + Table: table, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + }) + // Built from the same matchers as the production rule in + // ateomnet.ActorEgressRedirectRule, with a destination-port match added so + // the rule cannot fire again on the connection it just rewrote. + exprs := append(family.sourceEqual(family.actorIP), ateomnet.TCPProtocol()...) + exprs = append(exprs, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(uint16(targetPort))}, + &expr.Immediate{Register: 1, Data: binaryutil.BigEndian.PutUint16(uint16(redirectPort))}, + &expr.Redir{RegisterProtoMin: 1}, + ) + c.AddRule(&nftables.Rule{Table: table, Chain: chain, Exprs: exprs}) + if err := c.Flush(); err != nil { + t.Fatalf("installing the %s redirect: %v", family.name, err) + } +} diff --git a/manifests/ate-install/atenet-egress-with-sdsmint.yaml b/manifests/ate-install/atenet-egress-with-sdsmint.yaml index 72b3e19795..f283a0c34a 100644 --- a/manifests/ate-install/atenet-egress-with-sdsmint.yaml +++ b/manifests/ate-install/atenet-egress-with-sdsmint.yaml @@ -236,7 +236,7 @@ data: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: AUTO - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -323,7 +323,7 @@ data: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: AUTO - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -410,7 +410,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: AUTO # The MITM must not weaken upstream authentication. Envoy decrypted the # actor's TLS with a leaf of its own; it still sends the real SNI here # and still validates the real origin's certificate against the public @@ -453,7 +453,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: AUTO # Envoy refuses to build a dynamic forward proxy cluster without # auto_sni and auto_san_validation unless this is set, because for # the usual TLS case resolving the host from a header and then not diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fcb..64153ea6f5 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -37,12 +37,15 @@ data: envoy.yaml: | admin: address: - socket_address: { address: 0.0.0.0, port_value: 15000 } + # ipv4_compat is load-bearing: see --envoy-admin-address below. + socket_address: { address: "::", ipv4_compat: true, port_value: 15000 } static_resources: listeners: - name: egress address: - socket_address: { address: 0.0.0.0, port_value: 443 } + # ipv4_compat rather than a second socket: IPv4 peers arrive as + # ::ffff: and nothing here reads the peer -- identity is the cert. + socket_address: { address: "::", ipv4_compat: true, port_value: 443 } filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in @@ -142,7 +145,7 @@ data: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: AUTO - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -179,7 +182,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: AUTO --- apiVersion: apps/v1 kind: Deployment @@ -379,6 +382,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-egress ports: diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index e05e06efb7..462e2a8ad0 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -86,7 +86,9 @@ data: admin: address: socket_address: - address: 0.0.0.0 + # ipv4_compat is load-bearing: dataplane.go probes /ready over IPv4 loopback. + address: "::" + ipv4_compat: true port_value: 9901 node: @@ -354,6 +356,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-router ports: