diff --git a/benchmarking/routercap/.gitignore b/benchmarking/routercap/.gitignore new file mode 100644 index 000000000..04d348f1d --- /dev/null +++ b/benchmarking/routercap/.gitignore @@ -0,0 +1,13 @@ +# A run directory ships exactly one file: report.html, which embeds every +# chart inline. The raw data behind it (samples.jsonl, run.json, +# worker-cpu.jsonl per arm) and the regenerable summary.json/SVGs stay local +# to the machine that ran the sweep. +runs/** +!runs/*/ +!runs/*/report.html + +# profile.sh needs privileged node debug access that the shipped harness +# should not imply is routine. +profile.sh + +__pycache__/ diff --git a/benchmarking/routercap/README.md b/benchmarking/routercap/README.md new file mode 100644 index 000000000..280d32f3a --- /dev/null +++ b/benchmarking/routercap/README.md @@ -0,0 +1,326 @@ +# routercap + +Measures how much load one `atenet-router` pod absorbs before it stops +absorbing, and what the failure looks like when it does. The output is a time +series per run — `runs//report.html` — and the findings live in +[RESULTS.md](RESULTS.md). + +## Why this test exists + +Every actor request goes through `atenet-router`, and nobody had measured +what one pod can take. Without that figure there is no basis for the replica +count, for the CPU limits in +[atenet-router.yaml](../../manifests/ate-install/atenet-router.yaml), or for +an alert threshold — and no way to tell a regression from a busier week. + +A single "N QPS" number would not settle it either. What matters is the shape +at the edge: whether throughput plateaus or collapses, whether latency +degrades or cliffs, and whether the binding constraint is CPU at all. So the +harness walks a rising load ladder and records the whole curve, at several +Envoy CPU sizes. + +### Why not locust + +Locust-style load tests are closed-loop: each simulated user sends a request, +waits for the reply, then sends the next. The moment the router slows down, +the users slow down with it — offered load sags exactly when the system is +most interesting, and the latency samples miss the worst moments because +fewer requests were in flight during them. The literature calls this +coordinated omission. A closed-loop test of this router would report a +flattering curve that bends where the client throttled, not where the router +failed. + +This harness is open-loop: a pacer fires requests on a fixed schedule whether +or not earlier ones have returned, so offered load stays the independent +variable all the way through a collapse. The repo's boomer/locust rig +(`cmd/benchmarking/boomer-glutton`) remains the right tool for +workload-shaped soak tests; it is the wrong instrument for finding a wall. + +## What this measures + +`atenet-router` is one pod with two containers. `envoy` is the data plane. +`atenet-router` is a Go sidecar acting as Envoy's ext_proc server: it +decides, per request, which worker the actor is on and resumes it if it is +not running. + +```mermaid +flowchart LR + gen["load generator
POST /ping, Host: actor"] + subgraph pod["atenet-router pod — the system under test"] + direction TB + envoy["envoy
data plane"] + side["atenet-router
ext_proc server"] + end + api["ate-api-server"] + wk["worker pods
atunnel ingress :443
then actor sandbox"] + gen -->|"1 · HTTP/1.1, keep-alive"| envoy + envoy <-->|"2 · gRPC ext_proc"| side + side -->|"3 · ResumeActor"| api + envoy -->|"4 · mTLS to x-ate-original-dst"| wk +``` + +Envoy holds the request open across step 2, so every in-flight request +occupies one ext_proc slot and, at step 4, one upstream connection and one +source port. That is why concurrency is a measured series and not an +afterthought. + +### The latency + +End-to-end client-observed latency of one `/ping` request, timed from when +the pacer scheduled it to be sent — not from when it left the socket. +Concretely, the clock covers, in order: + +* any wait inside the generator (the request was due but blocked: no idle + connection, dial in progress). This wait is counted on purpose: at load it + usually means the router has not answered the requests already on the + connections, and a real client's request queues in its own pool the same + way, +* the TCP/TLS dial if a fresh connection was needed, +* Envoy's handling, the ext_proc call to the sidecar (warm actor resume + included), the round trip to the worker, +* the response coming back and being read. + +Starting the clock at the scheduled time avoids coordinated omission, the +standard way load tests lie: a clock that starts at the actual send never +measures a stall, because nothing is sent during one. Here, a request that +was due while the router stalled carries the whole stall in its latency. +Timeouts count their full elapsed time instead of vanishing, and percentiles +are computed from raw per-request samples, not histogram estimates. + +## What this doesn't measure + +**Cold actor starts.** Every actor is created and resumed before the ladder +begins; a cold resume takes ~3.8 s and would otherwise land inside the first +rung as router latency. The warm per-request control-plane lookup stays in +the path (it is part of every real request) and is reported separately as +`resume` — 0.7-1.5 ms in healthy windows. + +**DNS and kube-proxy.** In production a client resolves the actor's hostname +through ate's CoreDNS (which always answers with the router Service's +ClusterIP) and kube-proxy picks a router pod per TCP connection. This harness +dials router pod IPs directly, skipping both hops, so a wall indicts Envoy +with zero doubt — not conntrack, not kube-dns, not a NAT rule. What the +skipped hops would add is small and knowable (a per-connection DNAT costing +microseconds, a DNS lookup per dial); what they would cost the measurement is +attribution. + +```mermaid +flowchart TD + C["client
myactor.myspace.actors...ate.dev"] + C -->|"DNS query"| KD["kube-dns → ate CoreDNS
always answers: router Service ClusterIP"] + KD --> C + C -->|"TCP connect to ClusterIP
Host: myactor.myspace..."| KP["kube-proxy
picks ONE pod per connection
← replica balancing happens here"] + KP --> P1["router pod 1"] + KP --> P2["router pod 2"] + P1 --> W["workers"] + P2 --> W +``` + +When a run drives more than one router replica, the harness balances across +them itself, round-robin with each actor stuck to one replica — a cleaner split than +kube-proxy's random per-connection assignment, so multi-replica numbers are +an upper bound with a caveat. The boomer rig dials the Service DNS name, so +it too skips ate's CoreDNS but does pass through kube-proxy; only a real +client exercises the full path. + +## What this produces + +A run directory holds three raw-data files per arm — `samples.jsonl` (the +windows), `run.json` (the header) and `worker-cpu.jsonl` (per-thread CPU) — +plus the artifacts charts.py renders from them: `report.html` (every chart +embedded), the standalone SVGs, and `summary.json` (per-rung aggregates in +machine-readable form, for CI gates). Only `report.html` is committed; the +data behind it stays on the machine that ran the sweep. Debug +files (the rendered Job manifest, the binary's stderr, the thread sampler's +raw dumps) survive only when an arm fails. Six series per arm, all computed over +the same wall-clock window, so a vertical line through the chart panels is +one moment: + +| Series | What it is | Why read it | +|---|---|---| +| offered QPS | requests the pacer *scheduled* | the independent variable, taken from the schedule so a struggling generator cannot redefine the x-axis | +| latency p50, p95 | client-observed, from scheduled send time | p50 is the healthy-path cost; p95 is where degradation shows first | +| per-hop share | the mean request split across before-Envoy, Envoy, sidecar, worker | says *which* hop the latency is in | +| router CPU | cores, `envoy` and `atenet-router` separately | says whether the ceiling is CPU at all | +| per-thread CPU | the hottest and the mean Envoy worker thread, 0-1 axis | one thread can saturate while the container average reads idle; lines hugging is balance, a gap is skew | +| router memory | working set, both containers | says whether a plateau is a slow leak | + +The throughput panel carries four companions — achieved, success, in-flight +and the generator's connection pool — because the gaps between them are the +finding: achieved against offered says whether the router kept up, and a pool +step marks a window where a stall made the generator dial thousands of fresh +connections. + +### Reading the per-thread CPU panel + +Envoy assigns each connection to one worker thread for life, and container +CPU is the sum across threads, so one drowning thread can hide inside an +idle-looking total. The panel plots the busiest single thread against the +per-thread mean on a 0-1 axis: + +* lines hugging: load is balanced across threads, +* a gap opening: skew, one thread doing far more than its share, +* hottest at the 1.0 line: that thread is saturated, and requests assigned + to it queue no matter how idle the container looks. + +### Reading the per-hop share panel + +Each window's mean request is split into four spans that do not overlap and +sum to the whole; the panel draws each as a percentage of that window's mean: + +``` +100% ┌────────────────┐ ─┐ ─┐ + │ worker leg │ │ measured: │ + ├────────────────┤ │ upstream_rq_time │ + │ sidecar │ │ measured: ├─ in-Envoy time + ├────────────────┤ │ route.duration │ (downstream_rq_time, + │ Envoy itself │ │ residual │ measured) + ├────────────────┤ ─┘ ─┘ + │ before Envoy │ residual ← the rig's share: generator + 0% └────────────────┘ queueing plus the dial + one window's mean request = 100% +``` + +Two rules are built into the drawing. The spans are means, not percentiles — +percentiles do not decompose. And the panel hatches itself wherever Envoy's +whole-millisecond rounding is worth more than 5% of the mean request, which +is most healthy windows; the split is for reading collapses. Raw milliseconds +for any window are in the hover readout and `samples.jsonl`. + +## Methodology + +### Load generation + +The pacer fires on a fixed tick. The generator measures itself: dispatch lag +(scheduled vs actual send) near zero means the x-axis is real. The transport +dials without a per-host cap on purpose — a cap would queue requests +internally, which is a closed loop wearing an open loop's clothes; the pool +size is plotted instead. The default ladder is 16 rungs, +1,000 QPS each, +45 s per rung with the first 10 s discarded so no window blends a rung's ramp +with its steady state. Load spreads over one pre-warmed actor per worker pod +(100 by default, 200 in the shipped runs). + +Six guards separate "the rig ran out" from "the router ran out": generator +CPU, dispatch lag, client keep-alive, client port headroom, per-worker +connection rate, and control-plane throttling. A fatal trip ends the arm and +marks it rig-limited. Thresholds and reasoning are in +[guards.go](../../internal/benchmarking/routercap/guards.go). Envoy's own +port-exhaustion and breaker counters are deliberately data, not guards — that +cliff is what the run came to measure. + +### Data collection + +CPU and memory come from cAdvisor on the kubelet — the only source with raw +cumulative counters, CFS accounting and a per-container timestamp together +(Envoy exports no process CPU counter; `metrics.k8s.io` pre-averages over a +window it picks). The sampler runs off cAdvisor's clock: it polls until the +router container's timestamp advances, and every number in a record — CPU, +memory, Envoy deltas, latency percentiles — is computed over exactly that +`[t0, t1)` interval. The ~10 s that costs is the honest resolution of any +container CPU figure on a kubelet-managed node; `t0`, `t1` and +`alignment_spread_ms` ship in every record so the claim is checkable. The +full argument is at the top of +[cadvisor.go](../../internal/benchmarking/routercap/cadvisor.go). + +### Cluster setup, and why + +`provision.sh` builds a dedicated cluster, `substrate-routercap`, with four +tainted node pools plus GKE's small untainted `default-pool`: + +| Pool | Nodes (default) | Runs | +|---|---|---| +| `router` | 1 × `c3-standard-88` | `atenet-router`, alone | +| `workers` | 2 × `c3-standard-88` | worker pods (the shipped runs used 4 nodes / 200 pods) | +| `loadgen` | 1 × `c3-standard-88` | the generator, alone | +| `system` | 1 × `c3-standard-88` | api-server, controller, dns, valkey | +| `default-pool` | 1 × `e2-standard-8` | GKE addons only | + +The isolation that matters is the node, not the QoS class. Only the router +containers get explicit CPU limits, because Envoy's limit is the variable +under test. A CPU limit is CFS quota, and quota does not partition the things +that bite at this scale — NIC queues, conntrack, L3, memory bandwidth — which +are all per node. The router node stays at 88 cores even though the largest +arm needs 16: the arm's limit should be the only thing constraining Envoy. + +### Layout + +| Path | What it is | +|---|---| +| `benchmarking/routercap/provision.sh` | one-shot cluster build | +| `benchmarking/routercap/run.sh` | sweep driver: patches the router per arm, launches the generator Job, demuxes its output into the run directory | +| `benchmarking/routercap/common.sh` | shared config and helpers for the two scripts | +| `benchmarking/routercap/charts.py` | renders `report.html`, SVGs and `summary.json` from raw data only | +| `benchmarking/routercap/demux.py` | splits the Job's stdout stream (data) from stderr (logs) into per-arm files | +| `benchmarking/routercap/threads.sh` + `threads.py` | per-thread CPU sampler (ephemeral debug container reading /proc) and its parser | +| `benchmarking/routercap/manifests/` | the generator Job template | +| `benchmarking/routercap/runs/` | one directory per run; raw data is committed, HTML/SVG are regenerated | +| `cmd/benchmarking/routercap/` | the generator binary | +| `internal/benchmarking/routercap/` | the library: pacer, sender, actor pool, cAdvisor windows, Envoy/sidecar scrapers, span math, guards, records | + +## How to reproduce + +### Prerequisites + +
+What you need before starting + +* `gcloud` authenticated against a project that can create GKE clusters and + `c3-standard-88` nodes in `us-central1-a`. +* The environment configuration sourced first (`source .ate-dev-env.sh`) + so `PROJECT_ID`, `KO_DOCKER_REPO`, etc. are set. +* `kubectl`, `ko` (via `hack/run-tool.sh`) and `python3` on the path. + +
+ +### Step 1: provision the cluster + +* Creates the `substrate-routercap` cluster with its tainted node pools. +* Installs the substrate control plane. +* Applies the worker pool: one worker pod per replica on the worker nodes. +* Pins each component to its pool. + +```bash +benchmarking/routercap/provision.sh +``` + +### Step 2: smoke-test the rig + +* Runs one small arm with 2 actors and 3 short rungs. Measures nothing. +* Proves the wiring: the router patch takes, the Job launches, the scrapes + and cAdvisor windows fill, and the run directory lands. + +```bash +benchmarking/routercap/run.sh --smoke +``` + +### Step 3: run the sweep + +* For each arm, `run.sh` patches the router (Envoy CPU limit and + `--concurrency`, which restarts the pod). +* Purges any actors an earlier arm left behind, then creates and + warm-resumes one actor per worker pod. +* Walks the ladder and writes `runs//arm-c/`. + +```bash +benchmarking/routercap/run.sh --arms "2 4 8" +``` + +### Step 4: regenerate charts (optional) + +* Re-renders `report.html`, the SVGs and `summary.json` from raw data only, + so any past run directory works, including one whose cluster is long gone. + +```bash +python3 benchmarking/routercap/charts.py benchmarking/routercap/runs/ +``` + +### Step 5: tear down + +* Deletes the cluster directly. +* Do not use `hack/teardown.sh`: it targets your dev cluster and revokes + project-level IAM shared with every other cluster in the project. + +```bash +gcloud container clusters delete substrate-routercap \ + --location=us-central1-a --project="${PROJECT_ID}" --quiet +``` diff --git a/benchmarking/routercap/charts.py b/benchmarking/routercap/charts.py new file mode 100755 index 000000000..808515040 --- /dev/null +++ b/benchmarking/routercap/charts.py @@ -0,0 +1,1401 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Renders the capacity run's charts from a run directory. + + python3 charts.py benchmarking/routercap/runs/ + +Reads only ``arm-*/samples.jsonl``, ``arm-*/worker-cpu.jsonl`` and ``arm-*/run.json`` +— never a log — so charts regenerate from any past run, including one whose +cluster is long gone. + +Standard library only, and the SVG is emitted by hand, so the harness runs in +automation without a plotting stack. + +Outputs, all in the run directory: + + timeseries-c.svg five stacked panels sharing one x-axis + report.html self-contained: headline cards, the charts, the table + summary.json the same numbers, machine-readable +""" + +import argparse +import datetime +import glob +import html +import json +import math +import os +import re +import sys + +# The repository root, for spelling user-facing paths the same way no matter +# what directory a command ran from. charts.py lives in benchmarking/routercap. +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Fixed per role, so the same container is the same colour in every chart and +# across every arm. +COLORS = { + "offered": "#0b0b0b", + "achieved": "#2a78d6", + "success": "#1baf7a", + "in_flight": "#eb6834", + "connections": "#4a3aa7", + "p50": "#1baf7a", + "p95": "#eda100", + "envoy": "#2a78d6", + "atenet-router": "#eb6834", + "loadgen": "#898781", +} + +# The hops of one request, bottom of the stack first: (legend label, field in +# Sample.spans, colour). Colours match the CPU panels — blue is envoy, orange +# the sidecar, grey the generator — and labels stay under about fifteen +# characters, which is what the right gutter holds at this font size. +SPAN_LAYERS = [ + ("before Envoy", "before_envoy_ms", "#898781"), + ("Envoy itself", "envoy_internal_ms", "#2a78d6"), + ("sidecar", "sidecar_ms", "#eb6834"), + ("worker leg", "worker_ms", "#1baf7a"), +] +ARM_COLORS = ["#2a78d6", "#1baf7a", "#eda100", "#dc2626", "#4a3aa7", "#0891b2"] + +RUNG_FILL = "#f3f4f6" +THRESHOLD_PORTS = "#d03b3b" + +# Tooltip text is monospace so the background rectangle can be sized from the +# longest line without measuring glyphs. 6.32px is the advance width of +# ui-monospace at 10.5px, rounded up. +TIP_FONT_PX = 10.5 +TIP_CHAR_PX = 6.32 +TIP_LINE_PX = 13.5 + + +# -------------------------------------------------------------------------- +# loading + + +def parse_time(s): + """Parses Go's RFC3339 output, whose sub-second field can be nanoseconds.""" + if not s: + return None + s = s.replace("Z", "+00:00") + if "." in s: + head, rest = s.split(".", 1) + frac, _, tz = rest.partition("+") + s = "%s.%s+%s" % (head, frac[:6], tz) if tz else "%s.%s" % (head, frac[:6]) + try: + return datetime.datetime.fromisoformat(s) + except ValueError: + return None + + +def read_jsonl(path): + out = [] + if not os.path.exists(path): + return out + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + # A run killed mid-write leaves a torn final line; everything + # before it is still good. + continue + return out + + +class Arm: + """One arm's directory, loaded.""" + + def __init__(self, path): + self.path = path + self.name = os.path.basename(path) + self.header = {} + hp = os.path.join(path, "run.json") + if os.path.exists(hp): + with open(hp, encoding="utf-8") as fh: + self.header = json.load(fh) + self.samples = read_jsonl(os.path.join(path, "samples.jsonl")) + for s in self.samples: + s["_t0"] = parse_time(s.get("t0")) + s["_t1"] = parse_time(s.get("t1")) + s["_t"] = parse_time(s.get("t")) + self.samples = [s for s in self.samples if s["_t"]] + # Kept on the instance, not left a local: the rung schedule is placed + # against the same zero the samples are. + self.origin = min((s["_t0"] for s in self.samples if s["_t0"]), default=None) + for s in self.samples: + o = self.origin + s["_x"] = (s["_t"] - o).total_seconds() if o and s["_t"] else 0.0 + s["_x0"] = (s["_t0"] - o).total_seconds() if o and s["_t0"] else 0.0 + s["_x1"] = (s["_t1"] - o).total_seconds() if o and s["_t1"] else 0.0 + # Per-worker-thread CPU from the /proc sampler, when the run carried + # one. Its timestamps are unix epochs stamped by busybox, not ISO + # strings. + self.worker_cpu = read_jsonl(os.path.join(path, "worker-cpu.jsonl")) + if self.origin: + oe = self.origin.timestamp() + for w in self.worker_cpu: + w["_x"] = (w.get("t0", 0) + w.get("t1", 0)) / 2.0 - oe + self.worker_cpu = [w for w in self.worker_cpu if w["_x"] > 0] + else: + self.worker_cpu = [] + self.rungs = self._schedule() + + def _schedule(self): + """The ladder as the generator planned it: exact rung boundaries. + + run.json records what the pacer actually ran; sample windows cannot + substitute, being cut on cAdvisor's clock with no shared rung edge. + """ + results = self.header.get("results") or [] + if not results: + return [] + res = next((r for r in results if r.get("arm_cores") == self.cores), results[0]) + out = [] + for r in res.get("rungs") or []: + start = parse_time(r.get("start_at")) + if not (start and self.origin): + continue + x0 = (start - self.origin).total_seconds() + out.append({ + "index": r.get("index", 0), + "qps": r.get("rate_qps", 0), + "x0": x0, + # hold and warmup are Go durations, so nanoseconds. + "x1": x0 + (r.get("hold") or 0) / 1e9, + "warmup_s": (r.get("warmup") or 0) / 1e9, + }) + return out + + @property + def cores(self): + if self.samples: + return self.samples[0].get("arm_cores", 0) + return (self.header.get("arm_cores") or [0])[0] + + @property + def replicas(self): + # More than one router pod means the arm's shape describes each pod, + # not the tier: capacity-per-core and the label both have to say so. + # The header's router_pods list is the record of it. + return max(len(self.header.get("router_pods") or []), 1) + + @property + def label(self): + # The pass suffix survives only so that a run directory from before the + # two-pass mode was removed still labels its two same-size arms apart. + p = self.samples[0].get("pass", 1) if self.samples else 1 + out = "%dc" % self.cores if p <= 1 else "%dc p%d" % (self.cores, p) + if self.replicas > 1: + out = "%d×%s" % (self.replicas, out) + # Everything after arm-c in the directory name is tags: a t tag + # is the thread count and reads as 8c/2t. Any other tag is carried + # verbatim, so a merged-in diagnostic arm stays distinguishable from + # the real arm of the same size. + tagged = False + for tag in re.findall(r"-([a-z0-9]+)", self.name.replace("arm-", "", 1)): + if re.fullmatch(r"\d+t", tag): + out += "/" + tag + elif re.fullmatch(r"x\d+", tag): + pass # replica count, already on the label from the header + else: + out += " " + tag + tagged = True + if tagged: + return out + # No tags: the samples decide. An arm whose measured --concurrency + # differs from its core count (run.sh RC_CONCURRENCY) says so, or its + # chart is indistinguishable from the real arm. + threads = 0 + for s in self.samples: + c = (s.get("envoy") or {}).get("concurrency") or 0 + if c: + threads = int(c) + break + if threads and threads != self.cores: + out += "/%dt" % threads + return out + + def measured(self): + """Non-warmup samples: the ones an analysis is entitled to summarize.""" + return [s for s in self.samples if not s.get("warmup")] + + +def load_run(run_dir): + arms = [] + for path in sorted(glob.glob(os.path.join(run_dir, "arm-*"))): + if not os.path.isdir(path): + continue + a = Arm(path) + if a.samples: + arms.append(a) + else: + print("[charts] %s has no samples; skipping" % a.name, file=sys.stderr) + # Cores, then thread count, then name — numerically, so a thread ladder + # reads 2t, 4t, 8t, 16t. An arm without a -Nt suffix sorts at its own + # thread count, placing 8c between 8c-4t and 8c-16t. + def _threads(a): + m = re.search(r"-(\d+)t$", a.name) + return int(m.group(1)) if m else a.cores + arms.sort(key=lambda a: (a.cores, _threads(a), a.name)) + return arms + + +# -------------------------------------------------------------------------- +# accessors — one place that knows the record shape + + +def load_of(s, *keys, default=0.0): + v = s.get("load") or {} + for k in keys: + v = (v or {}).get(k) + if v is None: + return default + return v + + +def client_of(s, field, default=0.0): + v = (s.get("client") or {}).get(field) + return default if v is None else v + + +def container_of(s, role, field, default=0.0): + c = (s.get("containers") or {}).get(role) or {} + v = c.get(field) + return default if v is None else v + + +# Above this, Envoy's whole-millisecond rounding is large enough to flip which +# hop looks largest. The record carries the ratio; where to stop trusting it +# is a presentation decision and lives here. +COARSE_SHARE = 0.05 + + +def merge_spans(ranges): + """Overlapping or touching [x0, x1) intervals, merged and sorted.""" + out = [] + for x0, x1 in sorted(ranges): + if out and x0 <= out[-1][1]: + out[-1] = (out[-1][0], max(out[-1][1], x1)) + else: + out.append((x0, x1)) + return out + + +def span_shares(s): + """The four hops of this window as percentages, or None if unmeasured. + + Negative spans are floored at zero and the rest renormalised to 100 — the + chart's doing, not the collector's. The raw milliseconds, negative or + not, stay in the hover readout. + """ + sp = s.get("spans") or {} + if not sp.get("measured"): + return None + vals = [max(sp.get(k) or 0.0, 0.0) for _, k, _ in SPAN_LAYERS] + total = sum(vals) + if total <= 0: + return None + return [v / total * 100.0 for v in vals] + + +def has_container(s, role): + """True when this window actually measured role. + + cAdvisor can close a window before a container's sample ticked, leaving + container_of at its 0.0 default — a spike to the floor that reads as "the + sidecar stopped working". Series filter on this so an unsampled window is + a gap, not a fabricated zero. + """ + return ((s.get("containers") or {}).get(role) or {}).get("cpu_cores") is not None + + +# -------------------------------------------------------------------------- +# a very small SVG plotter + + +class Axis: + """Maps data values onto pixels, linearly or on a log scale.""" + + def __init__(self, lo, hi, px0, px1, log=False): + self.log = log and lo > 0 + if self.log: + lo, hi = math.log10(max(lo, 1e-9)), math.log10(max(hi, 1e-9)) + if hi <= lo: + hi = lo + 1.0 + self.lo, self.hi, self.px0, self.px1 = lo, hi, px0, px1 + + def __call__(self, v): + if self.log: + v = math.log10(max(v, 1e-9)) + f = (v - self.lo) / (self.hi - self.lo) + return self.px0 + f * (self.px1 - self.px0) + + def ticks(self, n=5): + if self.log: + out = [] + for e in range(int(math.floor(self.lo)), int(math.ceil(self.hi)) + 1): + out.append(10.0 ** e) + return [t for t in out if math.log10(t) >= self.lo - 1e-9] + step = nice_step((self.hi - self.lo) / max(n, 1)) + first = math.ceil(self.lo / step) * step + out, v = [], first + while v <= self.hi + 1e-9: + out.append(v) + v += step + return out + + +def nice_step(raw): + if raw <= 0: + return 1.0 + mag = 10.0 ** math.floor(math.log10(raw)) + for m in (1, 2, 2.5, 5, 10): + if raw <= m * mag: + return m * mag + return 10 * mag + + +def fmt(v): + if v == 0: + return "0" + a = abs(v) + if a >= 1e9: + return "%.1fG" % (v / 1e9) + if a >= 1e6: + return "%.1fM" % (v / 1e6) + if a >= 1e4: + return "%.0fk" % (v / 1e3) + if a >= 100: + return "%.0f" % v + if a >= 1: + return "%.1f" % v + return "%.3g" % v + + +# fmt() abbreviates above 10,000, which is right on an axis and wrong wherever +# the exact figure is the point: "11k qps sustainable" and "20k in flight" both +# hide the digits a reader would want to compare against another number. +def comma(v): + return "{:,}".format(int(round(v))) + + +def esc(s): + return html.escape(str(s), quote=True) + + +# SVG has no text wrapping, so a long silently runs off the canvas +# instead of reflowing. Greedy packing against a character budget is enough +# here: the notes are one font, one size, and prose rather than data, so a +# character count is a close enough proxy for width and errs on the safe side +# for the mostly-lowercase text it is used on. +def wrap(text, budget): + lines, cur = [], "" + for word in text.split(): + if cur and len(cur) + 1 + len(word) > budget: + lines.append(cur) + cur = word + else: + cur = word if not cur else cur + " " + word + if cur: + lines.append(cur) + return lines + + +class Panel: + """One plot area inside a chart.""" + + # note is a single line under the panel title defining that panel's + # series. Words next to the lines they describe get read, and they travel + # with the SVG when it is embedded without the surrounding page. + def __init__(self, title, ylabel, log=False, note="", fixed=None): + self.title, self.ylabel, self.log, self.note = title, ylabel, log, note + # fixed pins the y range instead of deriving it from the data, for a + # panel whose axis means something on its own — a percentage stack has + # to run 0 to 100 or the bands stop being shares. + self.fixed = fixed + self.series = [] # (label, color, [(x, y, [tooltip lines])]) + self.bands = [] # (x0, x1, fill) + self.hlines = [] # (y, color, label) + self.rung_labels = [] # (x_center, text) + # Hatched spans drawn over everything else, for a stretch of x where the + # data is there but should not be read. Over, not under: the point is to + # obscure slightly. + self.hatch = [] # (x0, x1) + # A stacked area, drawn as filled polygons rather than lines. + # stack_layers is bottom-first; stack_segs is contiguous runs of + # (x, [value per layer]), split so a band is never drawn across + # windows that were never measured. + self.stack_layers = [] # (label, color) + self.stack_segs = [] # [[(x, [v...]), ...], ...] + # hovers replaces per-point tooltips with one full-height column per + # x: the tooltip is written once instead of once per series, and the + # hover target is the whole column instead of a 2.6px dot. + self.hovers = [] # (x, [tooltip lines]) + + def add(self, label, color, points): + if points: + self.series.append((label, color, points)) + + def stack(self, layers, segs): + self.stack_layers = layers + self.stack_segs = [s for s in segs if s] + + def ymax(self): + vs = [p[1] for _, _, pts in self.series for p in pts] + vs += [y for y, _, _ in self.hlines] + return max(vs) if vs else 1.0 + + def ymin(self): + vs = [p[1] for _, _, pts in self.series for p in pts if p[1] > 0] + return min(vs) if vs else 0.1 + + +class Chart: + """A column of panels over one shared x-axis.""" + + # PAD_T clears the chart title and subtitle above the first panel, and + # PANEL_GAP clears each panel's own title plus up to NOTE_LINES wrapped + # lines of note. Both are sized from those constants rather than guessed. + PAD_L, PAD_R, PAD_B = 76, 150, 58 + PANEL_H, WIDTH = 200, 1180 + NOTE_LINES, NOTE_LEAD = 2, 15 + PANEL_GAP = 28 + NOTE_LINES * NOTE_LEAD + # 77 puts the first panel's title 18px below the chart subtitle's baseline + # at y=50. + PAD_T = 77 + NOTE_LINES * NOTE_LEAD + + # Tooltips are revealed by CSS rather than drawn by JavaScript: report.html + # and a bare SVG must both work from a file:// URL, and `display` toggling + # on an SVG group satisfies both. + CSS = ( + ".hg .tip{display:none;pointer-events:none}" + ".hg:hover .tip{display:block}" + ".hit{fill:#1d4ed8;fill-opacity:0}" + ".hg:hover .hit{fill-opacity:0.07}" + ".tipbg{fill:#0b0b0b;fill-opacity:0.5;stroke:#374151}" + ".tiptx{fill:#f9fafb;font-family:ui-monospace,SFMono-Regular,Menlo,monospace;" + "font-size:%.1fpx;white-space:pre}" + ) % TIP_FONT_PX + + def __init__(self, title, subtitle, xlabel): + self.title, self.subtitle, self.xlabel = title, subtitle, xlabel + self.panels = [] + + def panel(self, *a, **kw): + p = Panel(*a, **kw) + self.panels.append(p) + return p + + def render(self): + n = len(self.panels) + height = self.PAD_T + n * self.PANEL_H + (n - 1) * self.PANEL_GAP + self.PAD_B + xs = [p[0] for pan in self.panels for _, _, pts in pan.series for p in pts] + xs += [b[0] for pan in self.panels for b in pan.bands] + xs += [b[1] for pan in self.panels for b in pan.bands] + xs += [x for pan in self.panels for seg in pan.stack_segs for x, _ in seg] + xlo, xhi = (min(xs), max(xs)) if xs else (0.0, 1.0) + ax = Axis(xlo, xhi, self.PAD_L, self.WIDTH - self.PAD_R) + + o = [ + '' + % (self.WIDTH, height, self.WIDTH, height), + '', + "" % self.CSS, + '' + '' + "", + '%s' + % (self.PAD_L, esc(self.title)), + '%s' + % (self.PAD_L, esc(self.subtitle)), + ] + + body, tips = [], [] + for i, pan in enumerate(self.panels): + top = self.PAD_T + i * (self.PANEL_H + self.PANEL_GAP) + b, t = self._panel(pan, ax, top, top + self.PANEL_H, i == n - 1, height) + body += b + tips += t + # Tooltips last: SVG has no z-index, so painter's order is the only + # stacking control there is. + o += body + tips + o.append("") + return "\n".join(o) + + def _tooltip(self, lines, anchor_x, panel_top, height, klass="tip"): + """A hidden group that a :hover on the enclosing .hg reveals.""" + w = max(len(l) for l in lines) * TIP_CHAR_PX + 18 + h = len(lines) * TIP_LINE_PX + 13 + # Right of the cursor by default, flipped left when that would run off + # the edge, then clamped so a tooltip near a corner stays whole. + tx = anchor_x + 15 + if tx + w > self.WIDTH - 6: + tx = anchor_x - 15 - w + tx = max(6.0, min(tx, self.WIDTH - w - 6)) + ty = max(6.0, min(panel_top + 8.0, height - h - 6)) + spans = "".join( + '%s' % (tx + 9, 0 if i == 0 else TIP_LINE_PX, esc(l)) + for i, l in enumerate(lines)) + # display="none" duplicates the stylesheet on purpose: GitHub sanitises + # +
+
+

atenet-router capacity

+
Generated by running python3 %(cmd)s
+
+ +

Headline

+
Sustainable is the highest rung whose measured windows, summed, kept both completed and +successful requests within 1%% of what was offered, with no window's median above 100 ms.
+
%(cards)s
+ +

Legend

+
%(legend)s
+ +

Per arm, over time

+
One chart per arm. The panels share one x-axis and one set of window boundaries, so a +vertical line through them is a single interval. Hover any column for that window's full numbers.
+%(timeseries)s + +

Every measured rung

+
Warmup windows excluded. Latency, in-flight and memory are the worst window in the rung; +throughput and CPU are the mean across its windows. The highlighted row is each arm's last sustained rung.
+ + +%(rows)s
armrungofferedachievedsuccesssuccess %%p50 msp95 msin-flightenvoy c meansidecar c meanenvoy mem
+ +
+""" % { + "started": esc(str(hdr.get("started_at") or "")[:10] or "undated"), + "cluster": esc(hdr.get("cluster", "?")), + "machine": esc(hdr.get("machine_type", "?")), + # The regeneration command, spelled from the repo root regardless of + # where this invocation ran: both paths are recomputed relative to the + # root, which sits two levels above this file. + "cmd": esc("%s %s" % ( + os.path.relpath(os.path.abspath(__file__), REPO_ROOT), + os.path.relpath(os.path.abspath(run_dir), REPO_ROOT))), + "cards": armcards, + "legend": legend, + "timeseries": "".join('
%s
' % s for s in charts["timeseries"]), + "rows": "".join(rows), + } + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("run_dir", help="A run directory containing arm-*/ subdirectories.") + args = ap.parse_args() + + arms = load_run(args.run_dir) + if not arms: + print("[charts] no arm directories with samples under %s" % args.run_dir, file=sys.stderr) + return 1 + + charts = {"timeseries": []} + for arm in arms: + svg = timeseries_chart(arm) + path = os.path.join(args.run_dir, "timeseries-%s.svg" % arm.name.replace("arm-", "")) + with open(path, "w", encoding="utf-8") as fh: + fh.write(svg) + charts["timeseries"].append(svg) + + summary = summarize(arms) + with open(os.path.join(args.run_dir, "summary.json"), "w", encoding="utf-8") as fh: + json.dump(summary, fh, indent=2) + fh.write("\n") + with open(os.path.join(args.run_dir, "report.html"), "w", encoding="utf-8") as fh: + fh.write(report_html(args.run_dir, arms, summary, charts)) + + for a in summary["arms"]: + flag = " GUARDS: %s" % ", ".join(t["guard"] for t in a["guard_trips"]) if a["guard_trips"] else "" + # "≥" marks an arm that held its ladder's top rung: a floor, not a wall. + qual = "≥" if a.get("ladder_topped_out") else " " + print("[charts] %-14s sustainable %s%8s qps %7.0f qps/core peak in-flight %s%s" + % (a["name"], qual, fmt(a["sustainable_qps"]), a["qps_per_core"], fmt(a["peak_in_flight"]), flag), + file=sys.stderr) + print("[charts] wrote report.html, summary.json and %d SVGs to %s" + % (len(arms), args.run_dir), file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarking/routercap/common.sh b/benchmarking/routercap/common.sh new file mode 100644 index 000000000..4e79e05f8 --- /dev/null +++ b/benchmarking/routercap/common.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared settings and helpers for the atenet-router capacity harness. +# Sourced by provision.sh and run.sh; not executable on its own. + +# shellcheck disable=SC2034 # RC_* settings are consumed by the scripts that source this file + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +RC_DIR="${ROOT}/benchmarking/routercap" + +# The cluster this harness is allowed to touch; every destructive step checks +# it, so an inherited KUBECONFIG cannot point it at someone else's work. +RC_CLUSTER="${ROUTERCAP_CLUSTER:-substrate-routercap}" +RC_LOCATION="${ROUTERCAP_CLUSTER_LOCATION:-us-central1-a}" + +# A stockout surfaces as a raw gcloud error rather than a silent fallback to +# a smaller node, and whatever type is actually used is recorded in run.json. +# The router node is deliberately oversized so the only scarce resource on it +# is the one under test; provision.sh asserts the largest arm fits. +RC_MACHINE_TYPE="${ROUTERCAP_MACHINE_TYPE:-c3-standard-88}" +RC_SYSTEM_MACHINE_TYPE="${ROUTERCAP_SYSTEM_MACHINE_TYPE:-c3-standard-88}" +RC_WORKER_NODES="${ROUTERCAP_WORKER_NODES:-2}" + +# Its own kubeconfig, never the caller's. provision.sh and run.sh both export +# this, so neither can move anyone else's current context. +RC_KUBECONFIG="${ROUTERCAP_KUBECONFIG:-${HOME}/.kube/substrate-routercap.config}" + +# Namespaces and selectors, shared between the scripts and the binary's flag +# defaults. +RC_ROUTER_NS="ate-system" +RC_ROUTER_SELECTOR="app=atenet-router" +RC_WORKER_NS="benchmark-workloads" +RC_WORKER_POOL="benchmark-ateom" +RC_WORKER_SELECTOR="ate.dev/worker-pool" +RC_JOB_NS="benchmarking" +RC_JOB_SA="benchmark-runner" + +# Node pool roles. The label and the taint carry the same key and value: the +# label attracts the pods that belong, the taint repels everything else. +RC_ROLE_KEY="ate.dev/role" +RC_POOL_ROUTER="router" +RC_POOL_SYSTEM="system" +RC_POOL_WORKERS="workers" +RC_POOL_LOADGEN="loadgen" + +# The system pool's taint is soft; the other three are hard. A hard taint on +# system deadlocks the install: ate-system pods carry no toleration until +# provision.sh's pinning patch lands, so they need somewhere to schedule +# first. +# +# router, workers and loadgen stay NoSchedule: an uninvited pod on those nodes +# is exactly the contamination this harness exists to exclude. +RC_TAINT_SYSTEM="PreferNoSchedule" +RC_TAINT_HARD="NoSchedule" + +# The default arms: three Envoy CPU sizes, each running the same ladder. +# 2/4/8 doubles across the CPU-bound region — 2 well inside, 8 at the edge +# where cores stop being the binding constraint; see RESULTS.md for the +# sweeps that located it. +RC_ARMS_DEFAULT="2 4 8" + +# The Go sidecar's CPU, pinned across every arm so only the envoy container +# varies. Lives here because provision.sh needs the same number for its fit +# check, and two copies would drift. +RC_SIDECAR_CORES="${ROUTERCAP_SIDECAR_CORES:-8}" + +COLOR_CYAN='\033[1;36m' +COLOR_RED='\033[1;31m' +COLOR_RESET='\033[0m' + +rc::step() { echo -e "${COLOR_CYAN}[routercap] $*${COLOR_RESET}" >&2; } +rc::warn() { echo -e "${COLOR_RED}[routercap] $*${COLOR_RESET}" >&2; } +rc::die() { + rc::warn "$*" + exit 4 +} + +# rc::need checks for a binary on PATH. Checked up front rather than three +# minutes into a provision. +rc::need() { + local missing=() + for bin in "$@"; do + command -v "${bin}" >/dev/null 2>&1 || missing+=("${bin}") + done + if [[ ${#missing[@]} -gt 0 ]]; then + rc::die "missing required tools: ${missing[*]}" + fi +} + +# rc::kubectl runs kubectl against this harness's kubeconfig only. +rc::kubectl() { KUBECONFIG="${RC_KUBECONFIG}" kubectl "$@"; } + +# rc::assert_cluster refuses to act on anything but the harness's own cluster. +# The check is against the kubeconfig's current context, not against an +# environment variable, so exporting the right name at the wrong cluster does +# not get past it. +rc::assert_cluster() { + local ctx="" + ctx="$(KUBECONFIG="${RC_KUBECONFIG}" kubectl config current-context 2>/dev/null || true)" + if [[ -z "${ctx}" ]]; then + rc::die "no current context in ${RC_KUBECONFIG}; run provision.sh first" + fi + if [[ "${ctx}" != *"${RC_CLUSTER}"* ]]; then + rc::die "kubeconfig ${RC_KUBECONFIG} points at '${ctx}', which is not ${RC_CLUSTER}; refusing to touch it" + fi +} + +# rc::env sources the repo's dev env for PROJECT_ID / KO_DOCKER_REPO / +# BUCKET_NAME, then overrides the cluster coordinates with this harness's own. +# The install scripts read CLUSTER_NAME and CLUSTER_LOCATION from the +# environment; a developer's usual values would install substrate elsewhere. +rc::env() { + if [[ -f "${ROOT}/.ate-dev-env.sh" ]]; then + # shellcheck disable=SC1091 + source "${ROOT}/.ate-dev-env.sh" + fi + : "${PROJECT_ID:?PROJECT_ID must be set (put it in .ate-dev-env.sh)}" + : "${KO_DOCKER_REPO:?KO_DOCKER_REPO must be set (put it in .ate-dev-env.sh)}" + export CLUSTER_NAME="${RC_CLUSTER}" + export CLUSTER_LOCATION="${RC_LOCATION}" + export KUBECONFIG="${RC_KUBECONFIG}" + unset KUBECTL_CONTEXT || true +} + +# rc::tolerations emits the tolerations JSON for one role. +rc::tolerations() { + local role="$1" + # No "effect" field, deliberately: an empty effect matches every effect, so + # one toleration covers both the hard-tainted pools and the soft-tainted + # system pool. + printf '[{"key":"%s","operator":"Equal","value":"%s"}]' "${RC_ROLE_KEY}" "${role}" +} + +# rc::pin_workload pins one Deployment/DaemonSet/StatefulSet to a node pool: +# the nodeSelector puts the pod on the right pool, the toleration lets it past +# that pool's taint. Applied as a patch because manifests/ate-install is the +# product's and this placement is the experiment's. +rc::pin_workload() { + local kind="$1" ns="$2" name="$3" role="$4" + rc::kubectl -n "${ns}" patch "${kind}" "${name}" --type=strategic -p "$(cat </dev/null +} + +# rc::router_port_range reads the router pod's real ephemeral source-port +# range through an ephemeral busybox container targeting envoy specifically — +# the image is distroless, and /proc/sys is per network namespace. Prints +# "low high" on success and nothing on failure; an unreadable range is +# recorded as assumed rather than aborting the arm. +rc::router_port_range() { + local pod="$1" ns="$2" dbg="portrange-$$-${RANDOM}" + + # Created detached and read back from its logs, rather than attached: + # kubectl debug only attaches when given -i or -t, so the attached form + # returns nothing when run headless. + rc::kubectl -n "${ns}" debug "${pod}" \ + --target=envoy --image=busybox:1.36 --container="${dbg}" \ + --attach=false --quiet -- \ + cat /proc/sys/net/ipv4/ip_local_port_range >/dev/null 2>&1 || return 0 + + # Polled rather than slept on: the container exits in milliseconds once the + # image is local, but the first pull on a fresh node is not instant. + local out="" i + for (( i = 0; i < 60; i++ )); do + out="$(rc::kubectl -n "${ns}" logs "${pod}" -c "${dbg}" 2>/dev/null | tr -d '\r' | head -1 || true)" + [[ -n "${out}" ]] && break + sleep 1 + done + + # An ephemeral container cannot be removed, only its output read; the pod is + # replaced at the next arm anyway. + if [[ "${out}" =~ ^[0-9]+[[:space:]]+[0-9]+$ ]]; then + echo "${out}" + fi +} diff --git a/benchmarking/routercap/demux.py b/benchmarking/routercap/demux.py new file mode 100755 index 000000000..c0e0361b6 --- /dev/null +++ b/benchmarking/routercap/demux.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Splits one arm's pod output into the files a local run writes directly. + +The generator runs in a distroless container, so ``kubectl cp`` cannot +retrieve what it writes. Instead the binary tags every record with its +stream and writes them all to stdout, and this puts them back: + + kubectl logs -f job/... --all-containers | demux.py OUTDIR + + samples.jsonl aligned records: load, latency, CPU and memory over one + cAdvisor-defined window + run.json the run header + job.log everything else, which is the binary's own stderr + +1s generator-only records (stream "fine") are counted for the closing summary +line and discarded; nothing downstream reads them. Writes are flushed per +line so ``kubectl logs -f`` piped through here still behaves as a stream. +""" + +import argparse +import json +import os +import sys + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("outdir", help="Directory to write into; created if absent.") + ap.add_argument( + "--quiet", + action="store_true", + help="Do not echo a progress line per aligned record to stderr.", + ) + args = ap.parse_args() + + os.makedirs(args.outdir, exist_ok=True) + paths = { + "sample": os.path.join(args.outdir, "samples.jsonl"), + } + header_path = os.path.join(args.outdir, "run.json") + log_path = os.path.join(args.outdir, "job.log") + + files = {k: open(v, "w", encoding="utf-8") for k, v in paths.items()} + log = open(log_path, "w", encoding="utf-8") + counts = {"sample": 0, "fine": 0, "header": 0, "log": 0} + + try: + for line in sys.stdin: + line = line.rstrip("\n") + if not line: + continue + rec = None + if line.startswith("{"): + try: + rec = json.loads(line) + except json.JSONDecodeError: + rec = None + stream = rec.get("stream") if isinstance(rec, dict) else None + if stream in files: + # Unwrap: what lands in samples.jsonl is byte-identical in shape + # to what a local --output-dir run writes, so charts.py cannot + # tell the two apart and does not have to. + files[stream].write(json.dumps(rec["record"], separators=(",", ":")) + "\n") + files[stream].flush() + counts[stream] += 1 + if stream == "sample" and not args.quiet: + progress(rec["record"]) + elif stream == "fine": + counts["fine"] += 1 + elif stream == "header": + with open(header_path, "w", encoding="utf-8") as fh: + json.dump(rec["record"], fh, indent=2) + fh.write("\n") + counts["header"] += 1 + else: + log.write(line + "\n") + log.flush() + counts["log"] += 1 + finally: + for f in files.values(): + f.close() + log.close() + + print( + "[demux] {sample} samples, {fine} fine, {header} header, {log} log lines -> {d}".format( + d=args.outdir, **counts + ), + file=sys.stderr, + ) + # A run whose header never arrived is one nobody can interpret later, so + # say so loudly rather than leave a directory that looks complete. + if counts["header"] == 0: + print("[demux] WARNING: no run header in this arm's output", file=sys.stderr) + return 1 + return 0 + + +def progress(rec: dict) -> None: + """Echoes one aligned record as a human-readable line.""" + load = rec.get("load") or {} + lat = load.get("latency") or {} + containers = rec.get("containers") or {} + + def cores(role: str) -> str: + """Cores for role, or a dash when this window never sampled it. + + cAdvisor can close a window before a given container has ticked; + printing 0.00c there would read as "went idle" when it means "nobody + looked". + """ + v = (containers.get(role) or {}).get("cpu_cores") + return " -" if v is None else "%5.2f" % v + + trips = ",".join(g.get("guard", "?") for g in (rec.get("guards") or [])) + print( + "[{arm:>3}c rung {rung:>2}{warm}] offered {off:>7.0f} achieved {ach:>7.0f} " + "inflight {inf:>6d} p50 {p50:>6.1f}ms p95 {p95:>7.1f}ms " + "envoy {envoy}c sidecar {side}c{guards}".format( + arm=rec.get("arm_cores", 0), + rung=rec.get("rung", 0), + warm="w" if rec.get("warmup") else " ", + off=load.get("offered_qps", 0.0), + ach=load.get("achieved_qps", 0.0), + inf=int(load.get("in_flight_max", 0)), + p50=lat.get("p50_ms", 0.0), + p95=lat.get("p95_ms", 0.0), + envoy=cores("envoy"), + side=cores("atenet-router"), + guards=" GUARD:" + trips if trips else "", + ), + file=sys.stderr, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarking/routercap/manifests/job.yaml.tmpl b/benchmarking/routercap/manifests/job.yaml.tmpl new file mode 100644 index 000000000..5ae0be99d --- /dev/null +++ b/benchmarking/routercap/manifests/job.yaml.tmpl @@ -0,0 +1,150 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# One arm of the capacity sweep. Rendered and applied by run.sh, which +# substitutes every dollar-brace placeholder below and refuses to apply the +# result if one is left over — so no comment here may contain literal +# placeholder syntax (that check is a plain grep over the rendered file). +# +# Shaped to match benchmarking/automation's runner-job.yaml.tmpl (same +# namespace, ServiceAccount and flags) so the orchestrator can drive it later. + +apiVersion: batch/v1 +kind: Job +metadata: + name: ${JOB_NAME} + namespace: benchmarking + labels: + app: routercap + test-name: ${NAME} + tag: ${TAG} + arm: "${ARM}" +spec: + # No retries. A rerun would start from a cold actor pool against a router + # that has already been loaded, which is a different experiment wearing the + # same label. + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + activeDeadlineSeconds: ${DEADLINE} + template: + metadata: + labels: + app: routercap + test-name: ${NAME} + tag: ${TAG} + arm: "${ARM}" + spec: + restartPolicy: Never + serviceAccountName: benchmark-runner + # The widest source-port range Linux allows, 2.3x the 28,232 default: + # this is the generator's hard ceiling on concurrent connections, and + # the wider range keeps its cliff past the router's. The binary reads + # the effective range from /proc at startup (the client_ports guard + # follows it); ip_local_port_range is a "safe" sysctl, namespaced to + # this pod. + securityContext: + sysctls: + - name: net.ipv4.ip_local_port_range + value: "1025 65535" + # Alone on its own node: sharing one with the router or the workers + # would fold the generator's packet processing into their measurement. + nodeSelector: + ${ROLE_KEY}: loadgen + tolerations: + - key: ${ROLE_KEY} + operator: Equal + value: loadgen + effect: NoSchedule + containers: + # Named routercap because cAdvisor reports containers by name and the + # binary watches its own container to measure the load generator itself. + - name: routercap + image: ${IMAGE} + imagePullPolicy: IfNotPresent + args: + - "--arm=${ARM}" + - "--expect-concurrency=${CONCURRENCY}" + - "--router-pods=${ROUTER_PODS}" + - "--pass=${PASS}" + - "--actors=${ACTORS}" + - "--start-qps=${START_QPS}" + - "--step-qps=${STEP_QPS}" + - "--rungs=${RUNGS}" + - "--hold=${HOLD}" + - "--warmup=${WARMUP}" + - "--port-range=${PORT_RANGE}" + - "--name=${NAME}" + - "--tag=${TAG}" + - "--git-sha=${GIT_SHA}" + - "--cluster=${CLUSTER}" + - "--location=${LOCATION}" + - "--machine-type=${MACHINE_TYPE}" + # Nothing can read files back out of a distroless container, so + # stdout is the one channel that survives; run.sh splits the tagged + # lines back into files. Logs go to stderr. + - "--records-to-stdout" + env: + # The binary identifies its own container from these to watch its own + # CPU; without them it refuses to start rather than silently disable + # that guard. + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: GOMAXPROCS + value: "${LOADGEN_CPU}" + resources: + # requests == limits, like everything else in the run: the CPU + # guard trips at a fraction of this limit, so a Burstable container + # would make the threshold mean nothing. + requests: + cpu: "${LOADGEN_CPU}" + memory: ${LOADGEN_MEMORY} + limits: + cpu: "${LOADGEN_CPU}" + memory: ${LOADGEN_MEMORY} + volumeMounts: + # Client identity presented to ateapi when warming and tearing down the + # actor pool. Same pair the locust runner mounts. + - name: servicedns-ca + mountPath: /run/servicedns-ca + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: ca.crt + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem diff --git a/benchmarking/routercap/manifests/rbac.yaml b/benchmarking/routercap/manifests/rbac.yaml new file mode 100644 index 000000000..9cd1ea01e --- /dev/null +++ b/benchmarking/routercap/manifests/rbac.yaml @@ -0,0 +1,64 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The generator's identity. Namespace and ServiceAccount names match +# benchmarking/automation's runner-job.yaml.tmpl so the same Job can later be +# driven by the orchestrator without a second set of bindings. +# +# Read-only, deliberately: everything that writes to the cluster is run.sh's +# job, running as whoever invoked it. Generating load and mutating the system +# under test are different privileges. + +apiVersion: v1 +kind: Namespace +metadata: + name: benchmarking +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: benchmark-runner + namespace: benchmarking +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: routercap-runner +rules: +# Resource usage comes from cAdvisor through the kubelet (see cadvisor.go for +# why not metrics.k8s.io), reached via the API server's node proxy rather than +# the kubelet directly. The run spans four node pools and one kubelet reports +# only its own node, so this covers every node, not just the router's. +- apiGroups: [""] + resources: ["nodes/proxy", "nodes/metrics"] + verbs: ["get"] +# Pod discovery: which pod is the router, what its IP is, and which worker +# pods exist. The worker count is read from the cluster because the +# per-worker connection-rate guard divides a cluster-wide rate by it. +- apiGroups: [""] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: routercap-runner +subjects: +- kind: ServiceAccount + name: benchmark-runner + namespace: benchmarking +roleRef: + kind: ClusterRole + name: routercap-runner + apiGroup: rbac.authorization.k8s.io diff --git a/benchmarking/routercap/provision.sh b/benchmarking/routercap/provision.sh new file mode 100755 index 000000000..0d8a1af29 --- /dev/null +++ b/benchmarking/routercap/provision.sh @@ -0,0 +1,367 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Brings up the dedicated cluster the atenet-router capacity run measures on, +# installs substrate, and pins every component to the node pool it belongs on. +# Idempotent: re-running repairs a half-provisioned cluster. +# +# Confirming the zone actually has capacity for the SUT machine type is an +# operator step taken beforehand — see the README. A stockout surfaces as the +# raw gcloud error: no probe, no retry, no silent fallback to a smaller node. + +# shellcheck source=benchmarking/routercap/common.sh +source "$(git rev-parse --show-toplevel)/benchmarking/routercap/common.sh" + +RC_WORKER_PODS="${ROUTERCAP_WORKER_PODS:-100}" +SKIP_INSTALL=false + +usage() { + cat <-docker.pkg.dev//. +ar_host="${KO_DOCKER_REPO%%/*}" +case "${ar_host}" in + gcr.io) ar_repo="gcr.io"; ar_loc="us" ;; + us.gcr.io) ar_repo="us.gcr.io"; ar_loc="us" ;; + eu.gcr.io) ar_repo="eu.gcr.io"; ar_loc="europe" ;; + asia.gcr.io) ar_repo="asia.gcr.io"; ar_loc="asia" ;; + *) ar_repo="$(echo "${KO_DOCKER_REPO}" | cut -d/ -f3)"; ar_loc="${ar_host%%-docker.pkg.dev}" ;; +esac + +has_pull_access() { + local roles + roles="$(gcloud projects get-iam-policy "${PROJECT_ID}" \ + --flatten='bindings[].members' --filter="bindings.members:${node_sa}" \ + --format='value(bindings.role)' 2>/dev/null || true)" + case "${roles}" in *artifactregistry.reader*) return 0 ;; esac + roles="$(gcloud artifacts repositories get-iam-policy "${ar_repo}" \ + --project="${PROJECT_ID}" --location="${ar_loc}" \ + --flatten='bindings[].members' --filter="bindings.members:${node_sa}" \ + --format='value(bindings.role)' 2>/dev/null || true)" + case "${roles}" in *artifactregistry.reader*) return 0 ;; esac + return 1 +} + +if ! has_pull_access; then + rc::die "the GKE node service account (${PROJECT_NUMBER}-compute@developer.gserviceaccount.com) has no roles/artifactregistry.reader on ${PROJECT_ID} or on the ${ar_repo} repository, so nodes cannot pull substrate images and every pod would land in ImagePullBackOff. Grant it with either: + + go run ./tools/setup-gcp create iam --gke-nodes --atelet --bucket-bindings --bucket \"\${BUCKET_NAME}\" + +or, if this project's IAM automation strips project-level bindings, the narrower repository-scoped form: + + gcloud artifacts repositories add-iam-policy-binding ${ar_repo} --location=${ar_loc} \\ + --member=${node_sa} --role=roles/artifactregistry.reader + +This script reports rather than grants: IAM here is shared with every cluster in the project." +fi + +# GKE caps a node at 110 pods and DaemonSets count toward that cap; asserted +# here rather than discovered with the pool already paid for. 10 is a +# generous allowance for GKE's own DaemonSets plus atelet. +per_node=$(( (RC_WORKER_PODS + RC_WORKER_NODES - 1) / RC_WORKER_NODES )) +if (( per_node + 10 > 110 )); then + rc::die "${RC_WORKER_PODS} worker pods over ${RC_WORKER_NODES} nodes is ${per_node}/node, past GKE's 110-per-node cap once DaemonSets are counted; raise ROUTERCAP_WORKER_NODES" +fi + +# --- cluster ----------------------------------------------------------------- + +cluster_exists() { + gcloud container clusters describe "${RC_CLUSTER}" \ + --location="${RC_LOCATION}" --project="${PROJECT_ID}" >/dev/null 2>&1 +} + +# substrate needs certificates.k8s.io/v1beta1 for PodCertificateRequest and +# ClusterTrustBundle, which is 1.36+, and the zone's default can be behind +# that. The version is resolved rather than defaulted: newest valid release +# at or above the floor. +RC_MIN_MINOR=36 +resolve_cluster_version() { + if [[ -n "${ROUTERCAP_CLUSTER_VERSION:-}" ]]; then + echo "${ROUTERCAP_CLUSTER_VERSION}" + return + fi + # validMasterVersions comes back newest-first, so the first one clearing the + # floor is the newest that does. Compared numerically rather than by regex: + # a pattern over version strings is how you end up rejecting 1.40. + gcloud container get-server-config \ + --location="${RC_LOCATION}" --project="${PROJECT_ID}" \ + --format="value(validMasterVersions)" 2>/dev/null \ + | tr ';' '\n' \ + | awk -F. -v floor="${RC_MIN_MINOR}" '$1 == 1 && $2 >= floor { print; exit }' +} + +if cluster_exists; then + rc::step "cluster ${RC_CLUSTER} already exists" +else + version="$(resolve_cluster_version)" + if [[ -z "${version}" ]]; then + rc::die "no GKE version >= 1.${RC_MIN_MINOR} offered in ${RC_LOCATION}; substrate needs certificates.k8s.io/v1beta1. Pin one with ROUTERCAP_CLUSTER_VERSION if you know better." + fi + rc::step "creating cluster ${RC_CLUSTER} in ${RC_LOCATION} on ${version}" + # The default pool stays small and untainted: GKE's own addons have to land + # somewhere. + gcloud container clusters create "${RC_CLUSTER}" \ + --project="${PROJECT_ID}" \ + --location="${RC_LOCATION}" \ + --cluster-version="${version}" \ + --num-nodes=1 \ + --machine-type=e2-standard-8 \ + --workload-pool="${PROJECT_ID}.svc.id.goog" \ + --enable-kubernetes-unstable-apis=certificates.k8s.io/v1beta1/podcertificaterequests,certificates.k8s.io/v1beta1/clustertrustbundles +fi + +# Pools are created router-first: it is the one pool that genuinely needs the +# large machine type, so a zone shortage surfaces before the other three have +# been paid for. + +# gcloud reports taint effects in the API's enum spelling, not the one you +# create them with, so a comparison has to translate. +taint_enum() { + case "$1" in + NoSchedule) echo NO_SCHEDULE ;; + PreferNoSchedule) echo PREFER_NO_SCHEDULE ;; + NoExecute) echo NO_EXECUTE ;; + *) rc::die "unknown taint effect: $1" ;; + esac +} + +ensure_pool() { + local name="$1" machine="$2" nodes="$3" role="$4" effect="$5" + if gcloud container node-pools describe "${name}" \ + --cluster="${RC_CLUSTER}" --location="${RC_LOCATION}" \ + --project="${PROJECT_ID}" >/dev/null 2>&1; then + # Reconcile the taint rather than just reporting the pool present: a pool + # created by an older revision of this script carries that revision's + # taint, and the system pool's effect is the difference between an install + # that converges and one that deadlocks. + local have want + have="$(gcloud container node-pools describe "${name}" \ + --cluster="${RC_CLUSTER}" --location="${RC_LOCATION}" --project="${PROJECT_ID}" \ + --format="value(config.taints[0].effect)" 2>/dev/null || true)" + want="$(taint_enum "${effect}")" + if [[ "${have}" != "${want}" ]]; then + rc::step "node pool ${name} exists with taint effect ${have:-}; updating to ${want}" + # --quiet because the update prompts to confirm replacing the pool's + # taints, and a provision run has to be unattended. Safe: the taints + # being replaced are the ones this same function wrote. + gcloud container node-pools update "${name}" --quiet \ + --cluster="${RC_CLUSTER}" --location="${RC_LOCATION}" --project="${PROJECT_ID}" \ + --node-taints="${RC_ROLE_KEY}=${role}:${effect}" + else + rc::step "node pool ${name} already exists" + fi + return + fi + rc::step "creating node pool ${name} (${nodes} x ${machine})" + gcloud container node-pools create "${name}" \ + --cluster="${RC_CLUSTER}" \ + --location="${RC_LOCATION}" \ + --project="${PROJECT_ID}" \ + --machine-type="${machine}" \ + --num-nodes="${nodes}" \ + --node-labels="${RC_ROLE_KEY}=${role}" \ + --node-taints="${RC_ROLE_KEY}=${role}:${effect}" +} + +ensure_pool "${RC_POOL_ROUTER}" "${RC_MACHINE_TYPE}" 1 "${RC_POOL_ROUTER}" "${RC_TAINT_HARD}" +ensure_pool "${RC_POOL_SYSTEM}" "${RC_SYSTEM_MACHINE_TYPE}" 1 "${RC_POOL_SYSTEM}" "${RC_TAINT_SYSTEM}" +ensure_pool "${RC_POOL_WORKERS}" "${RC_MACHINE_TYPE}" "${RC_WORKER_NODES}" "${RC_POOL_WORKERS}" "${RC_TAINT_HARD}" +ensure_pool "${RC_POOL_LOADGEN}" "${RC_MACHINE_TYPE}" 1 "${RC_POOL_LOADGEN}" "${RC_TAINT_HARD}" + +rc::step "fetching credentials into ${RC_KUBECONFIG}" +mkdir -p "$(dirname "${RC_KUBECONFIG}")" +KUBECONFIG="${RC_KUBECONFIG}" gcloud container clusters get-credentials "${RC_CLUSTER}" \ + --location="${RC_LOCATION}" --project="${PROJECT_ID}" +rc::assert_cluster + +# --- does the largest arm actually fit? --------------------------------------- +# +# If the biggest arm exceeds what the router node can allocate, the rollout +# does not fail loudly: the new pod sits Pending and the arm records the +# *previous* arm's CPU under the new arm's label. Caught here at provision +# time rather than forty minutes into a sweep. +# +# The DaemonSet allowance is a flat 3 cores: atelet requests 2, and GKE's own +# per-node DaemonSets come to well under 1. An allowance rather than a +# measurement, because atelet is not installed yet at this point. +rc::step "checking the largest arm fits the router node" +rc::kubectl wait --for=condition=Ready node \ + -l "${RC_ROLE_KEY}=${RC_POOL_ROUTER}" --timeout=10m >/dev/null + +router_alloc="$(rc::kubectl get node -l "${RC_ROLE_KEY}=${RC_POOL_ROUTER}" \ + -o jsonpath='{.items[0].status.allocatable.cpu}')" +# Allocatable is either plain cores ("88") or millicores ("87630m"). +if [[ "${router_alloc}" == *m ]]; then + alloc_m="${router_alloc%m}" +else + alloc_m=$(( router_alloc * 1000 )) +fi + +max_arm=0 +for arm in ${RC_ARMS_DEFAULT}; do + (( arm > max_arm )) && max_arm="${arm}" +done +need_m=$(( (max_arm + RC_SIDECAR_CORES + 3) * 1000 )) + +if (( need_m > alloc_m )); then + rc::die "router node (${RC_MACHINE_TYPE}) allocates ${alloc_m}m CPU, but the largest arm needs ${need_m}m (${max_arm} envoy + ${RC_SIDECAR_CORES} sidecar + 3 for DaemonSets). Use a larger ROUTERCAP_MACHINE_TYPE, or drop the top arm." +fi +rc::step "router node allocates ${alloc_m}m; largest arm needs ${need_m}m — fits with $(( (alloc_m - need_m) / 1000 )) cores spare" + +if [[ "${SKIP_INSTALL}" == "true" ]]; then + rc::step "--skip-install: stopping after the cluster and pools" + exit 0 +fi + +# --- substrate --------------------------------------------------------------- + +rc::step "installing substrate (hack/install-ate.sh --deploy-ate-system)" +# NO_DEV_ENV=1 because install-ate.sh re-sources .ate-dev-env.sh, which would +# undo rc::env's CLUSTER_NAME override and install substrate into whatever +# cluster the developer's file names. rc::env has already exported everything +# the install needs. +NO_DEV_ENV=1 "${ROOT}/hack/install-ate.sh" --deploy-ate-system + +# --- placement --------------------------------------------------------------- +# +# Node pinning is what isolates the measurement: a CPU limit is CFS quota, not +# core pinning, and does not partition run-queue delay, L3, memory bandwidth, +# the NIC or conntrack — all per node. Giving the router a node to itself +# partitions all of them at once. + +rc::step "pinning ate-system to its pools" +rc::pin_workload deployment "${RC_ROUTER_NS}" atenet-router "${RC_POOL_ROUTER}" +rc::pin_workload deployment "${RC_ROUTER_NS}" ate-api-server "${RC_POOL_SYSTEM}" +rc::pin_workload deployment "${RC_ROUTER_NS}" ate-controller "${RC_POOL_SYSTEM}" +rc::pin_workload deployment "${RC_ROUTER_NS}" dns "${RC_POOL_SYSTEM}" +rc::pin_workload statefulset "${RC_ROUTER_NS}" valkey-cluster "${RC_POOL_SYSTEM}" +rc::pin_workload deployment podcertificate-controller-system podcertificate-controller "${RC_POOL_SYSTEM}" + +# atelet is a DaemonSet and nothing under manifests/ate-install declares any +# tolerations, so without this it gets no worker node and no actor ever +# starts. It must run everywhere, so it tolerates all four taints. +rc::step "letting atelet onto every tainted pool" +rc::kubectl -n "${RC_ROUTER_NS}" patch daemonset atelet --type=strategic -p "$(cat </dev/null + +# --- workloads --------------------------------------------------------------- + +rc::step "deploying workloads (${RC_WORKER_PODS} worker pods)" +"${ROOT}/benchmarking/workloads/deploy.sh" --deploy --worker-count "${RC_WORKER_PODS}" + +# The shared workloads template carries no placement, so the pool is patched +# here rather than forked. Same ActorTemplate, same ateom image, different +# scheduling. +rc::step "pinning the worker pool" +rc::kubectl -n "${RC_WORKER_NS}" patch workerpool "${RC_WORKER_POOL}" --type=merge -p "$(cat </dev/null + +# --- the run's own namespace and RBAC ---------------------------------------- + +rc::step "applying the runner's namespace, ServiceAccount and RBAC" +rc::kubectl apply -f "${RC_DIR}/manifests/rbac.yaml" + +# --- checks ------------------------------------------------------------------ + +rc::step "waiting for ate-system to settle" +rc::kubectl -n "${RC_ROUTER_NS}" rollout status deployment/atenet-router --timeout=10m +rc::kubectl -n "${RC_ROUTER_NS}" rollout status deployment/ate-api-server --timeout=10m + +router_pod="$(rc::kubectl -n "${RC_ROUTER_NS}" get pod -l "${RC_ROUTER_SELECTOR}" -o jsonpath='{.items[0].metadata.name}')" +range="$(rc::router_port_range "${router_pod}" "${RC_ROUTER_NS}")" +if [[ -n "${range}" ]]; then + low="${range%%[[:space:]]*}"; high="${range##*[[:space:]]}" + rc::step "router ephemeral port range: ${low}-${high} ($((high - low + 1)) ports)" +else + rc::warn "could not read the router's ip_local_port_range; runs will record the Linux default as assumed" +fi + +# Derived rather than hardcoded, because the machine type is a variable and a +# stale dollar figure is worse than none. ~$0.05 per vCPU-hour is C3 on-demand +# in us-central1 with memory folded in — sizes the decision, not an invoice. +total_vcpu=$(( ${RC_MACHINE_TYPE##*-} * (2 + RC_WORKER_NODES) + ${RC_SYSTEM_MACHINE_TYPE##*-} )) + +cat >&2 <). + --tag T Run tag; defaults to the short commit, with -dirty if the tree is. + --image REF Skip the ko build and use this image. + --smoke One arm, 2 actors, 3 short rungs. Proves the rig, measures nothing. + --skip-charts Leave charts.py unrun. + -h, --help This. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --arms) shift; ARMS="$1" ;; + --arms=*) ARMS="${1#*=}" ;; + --actors) shift; ACTORS="$1" ;; + --actors=*) ACTORS="${1#*=}" ;; + --start-qps) shift; START_QPS="$1" ;; + --start-qps=*) START_QPS="${1#*=}" ;; + --step-qps) shift; STEP_QPS="$1" ;; + --step-qps=*) STEP_QPS="${1#*=}" ;; + --rungs) shift; RUNGS="$1" ;; + --rungs=*) RUNGS="${1#*=}" ;; + --hold) shift; HOLD_S="$1" ;; + --hold=*) HOLD_S="${1#*=}" ;; + --warmup) shift; WARMUP_S="$1" ;; + --warmup=*) WARMUP_S="${1#*=}" ;; + --output-dir) shift; OUTPUT_DIR="$1" ;; + --output-dir=*) OUTPUT_DIR="${1#*=}" ;; + --tag) shift; TAG="$1" ;; + --tag=*) TAG="${1#*=}" ;; + --image) shift; IMAGE="$1" ;; + --image=*) IMAGE="${1#*=}" ;; + --smoke) SMOKE=true ;; + --skip-charts) SKIP_CHARTS=true ;; + -h|--help) usage; exit 0 ;; + *) rc::die "unknown option: $1" ;; + esac + shift +done + +if [[ "${SMOKE}" == "true" ]]; then + # Proves the rig end to end; measures nothing. Three rungs is the fewest + # that gives the ladder a slope. + ARMS="${ARMS%% *}" + ACTORS=2 + RUNGS=3 + HOLD_S=30 + WARMUP_S=5 + START_QPS=200 + STEP_QPS=200 +fi + +rc::need kubectl git go python3 +rc::env +rc::assert_cluster + +GIT_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +if [[ -z "${TAG}" ]]; then + TAG="$(git -C "${ROOT}" rev-parse --short HEAD)" + if [[ -n "$(git -C "${ROOT}" status --porcelain)" ]]; then + # A run tagged with a clean commit that was not the code that ran is worse + # than no tag at all. + TAG="${TAG}-dirty" + fi +fi +if [[ -z "${OUTPUT_DIR}" ]]; then + OUTPUT_DIR="${RC_DIR}/runs/$(date -u +%Y%m%dT%H%M%SZ)" +fi +mkdir -p "${OUTPUT_DIR}" + +MACHINE_TYPE="$(rc::kubectl get nodes -l "${RC_ROLE_KEY}=${RC_POOL_ROUTER}" \ + -o jsonpath='{.items[0].metadata.labels.node\.kubernetes\.io/instance-type}' 2>/dev/null || true)" +: "${MACHINE_TYPE:=${RC_MACHINE_TYPE}}" + +# --- preflight --------------------------------------------------------------- + +rc::step "preflight" +worker_pods="$(rc::kubectl -n "${RC_WORKER_NS}" get pods -l "${RC_WORKER_SELECTOR}" \ + --field-selector=status.phase=Running -o name | wc -l | tr -d ' ')" +if (( worker_pods < ACTORS )); then + rc::die "${worker_pods} worker pods are Running but ${ACTORS} actors were asked for; one actor per pod is what keeps the per-worker connection-rate limit from binding before the concurrency limit" +fi +rc::step "${worker_pods} worker pods running" + +# A generator that does not fit its node sits Pending until the sweep times +# out. Checked here rather than in provision.sh because LOADGEN_CPU lives +# here and the loadgen pool can be resized between a provision and a run. +loadgen_alloc="$(rc::kubectl get node -l "${RC_ROLE_KEY}=${RC_POOL_LOADGEN}" \ + -o jsonpath='{.items[0].status.allocatable.cpu}' 2>/dev/null || true)" +if [[ -n "${loadgen_alloc}" ]]; then + if [[ "${loadgen_alloc}" == *m ]]; then lg_m="${loadgen_alloc%m}"; else lg_m=$(( loadgen_alloc * 1000 )); fi + # 3 cores for atelet and GKE's own DaemonSets, same allowance provision.sh uses. + if (( (LOADGEN_CPU + 3) * 1000 > lg_m )); then + rc::die "the generator's ${LOADGEN_CPU} cores do not fit the loadgen node (${lg_m}m allocatable, 3 cores reserved for DaemonSets); the Job would sit Pending. Lower LOADGEN_CPU in run.sh or grow the pool's machine type" + fi +fi + +rc::kubectl apply -f "${RC_DIR}/manifests/rbac.yaml" >/dev/null + +if [[ -z "${IMAGE}" ]]; then + rc::step "building the generator image" + ldflags=() + while IFS= read -r line || [[ -n "${line}" ]]; do + [[ -n "${line}" ]] && ldflags+=("--ldflags=${line}") + done < <(make -C "${ROOT}" ldflags) + # In a subshell at the repo root: ko resolves ./cmd/... against its own + # working directory. + IMAGE="$(cd "${ROOT}" && "${ROOT}/hack/run-tool.sh" ko build --platform=linux/amd64 \ + "${ldflags[@]}" ./cmd/benchmarking/routercap | tail -1)" +fi +rc::step "generator image: ${IMAGE}" + +# --- one arm ----------------------------------------------------------------- + +# patch_arm resizes the Envoy container and matches its --concurrency; the +# binary scrapes envoy_server_concurrency back and refuses a mismatch. +# RC_CONCURRENCY overrides only --concurrency (diagnostic runs); +# RC_ROUTER_PODS scales the replicas, and the next run scales back to 1. +# Recreate strategy = fresh pod per arm; the sidecar stays pinned so the two +# containers remain separable. +patch_arm() { + local arm="$1" patch="" threads="${RC_CONCURRENCY:-$1}" replicas="${RC_ROUTER_PODS:-1}" + patch="$(rc::kubectl -n "${RC_ROUTER_NS}" get deployment atenet-router -o json | python3 -c ' +import json, sys + +arm, sidecar, threads, replicas = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) +spec = json.load(sys.stdin)["spec"]["template"]["spec"] +# Replaces the whole strategy object, so any rollingUpdate block goes with it; +# leaving one behind alongside type Recreate is rejected by the API server. +ops = [{"op": "replace", "path": "/spec/strategy", "value": {"type": "Recreate"}}, + {"op": "replace", "path": "/spec/replicas", "value": replicas}] +seen = False +for i, c in enumerate(spec["containers"]): + envoy = c["name"] == "envoy" + cores = arm if envoy else sidecar + for field in ("requests", "limits"): + ops.append({"op": "replace", + "path": "/spec/template/spec/containers/%d/resources/%s/cpu" % (i, field), + "value": cores}) + if not envoy: + continue + seen = True + for key in ("command", "args"): + argv = c.get(key) or [] + if "--concurrency" in argv: + ops.append({"op": "replace", + "path": "/spec/template/spec/containers/%d/%s/%d" % (i, key, argv.index("--concurrency") + 1), + "value": threads}) + break + else: + sys.exit("the envoy container passes no --concurrency, so its worker threads cannot be kept in step with its CPU limit") +if not seen: + sys.exit("no container named envoy in the atenet-router Deployment") +json.dump(ops, sys.stdout) +' "${arm}" "${SIDECAR_CORES}" "${threads}" "${replicas}")" || return 4 + rc::kubectl -n "${RC_ROUTER_NS}" patch deployment atenet-router --type=json -p "${patch}" >/dev/null + rc::kubectl -n "${RC_ROUTER_NS}" rollout status deployment/atenet-router --timeout=10m +} + +# wait_pod_started blocks until the Job's pod is past Pending, so the log +# stream that follows starts at the first line rather than erroring out. +wait_pod_started() { + local job="$1" deadline=$((SECONDS + 600)) + while (( SECONDS < deadline )); do + local phase="" + phase="$(rc::kubectl -n "${RC_JOB_NS}" get pod -l "job-name=${job}" \ + -o jsonpath='{.items[0].status.phase}' 2>/dev/null || true)" + case "${phase}" in + Running|Succeeded|Failed) return 0 ;; + esac + sleep 2 + done + return 1 +} + +# job_exit_code reads the generator's own exit status. The binary distinguishes +# rig-limited from failed from interrupted, and collapsing that to "the Job +# failed" would throw away the only bit that says whether the number is usable. +job_exit_code() { + local job="$1" deadline=$((SECONDS + 300)) + while (( SECONDS < deadline )); do + local code="" + code="$(rc::kubectl -n "${RC_JOB_NS}" get pod -l "job-name=${job}" \ + -o jsonpath='{.items[0].status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || true)" + if [[ -n "${code}" ]]; then + echo "${code}" + return 0 + fi + sleep 2 + done + echo 1 +} + +run_arm() { + local arm="$1" + local dir="${OUTPUT_DIR}/arm-${arm}c" + mkdir -p "${dir}" + + rc::step "arm ${arm}c: patching the router" + # Checked explicitly: the sweep runs each arm with errexit off so one bad arm + # does not take the rest of the sweep with it, which means a failing call + # inside here does not unwind on its own. + if ! patch_arm "${arm}"; then + rc::warn "arm ${arm}c: could not resize the router; skipping rather than measuring the previous arm again under this arm's label" + return 4 + fi + + local pod + pod="$(rc::kubectl -n "${RC_ROUTER_NS}" get pod -l "${RC_ROUTER_SELECTOR}" \ + -o jsonpath='{.items[0].metadata.name}')" + + # Read per arm, not once: the rollout replaces the pod, and a range read from + # the previous pod is a number about a container that no longer exists. + local range="" + range="$(rc::router_port_range "${pod}" "${RC_ROUTER_NS}")" + if [[ -z "${range}" ]]; then + rc::warn "could not read ip_local_port_range from ${pod}; the header will say the default was assumed" + fi + local port_range="${range// /-}" + + local job + job="routercap-${arm}c-$(date -u +%H%M%S)" + local deadline=$(( RUNGS * HOLD_S + 900 )) + + rc::step "arm ${arm}c: launching ${job} (deadline ${deadline}s)" + sed \ + -e "s|\${JOB_NAME}|${job}|g" \ + -e "s|\${IMAGE}|${IMAGE}|g" \ + -e "s|\${ARM}|${arm}|g" \ + -e "s|\${CONCURRENCY}|${RC_CONCURRENCY:-${arm}}|g" \ + -e "s|\${ROUTER_PODS}|${RC_ROUTER_PODS:-1}|g" \ + -e "s|\${PASS}|1|g" \ + -e "s|\${ACTORS}|${ACTORS}|g" \ + -e "s|\${START_QPS}|${START_QPS}|g" \ + -e "s|\${STEP_QPS}|${STEP_QPS}|g" \ + -e "s|\${RUNGS}|${RUNGS}|g" \ + -e "s|\${HOLD}|${HOLD_S}s|g" \ + -e "s|\${WARMUP}|${WARMUP_S}s|g" \ + -e "s|\${PORT_RANGE}|${port_range}|g" \ + -e "s|\${NAME}|routercap|g" \ + -e "s|\${TAG}|${TAG}|g" \ + -e "s|\${GIT_SHA}|${GIT_SHA}|g" \ + -e "s|\${CLUSTER}|${RC_CLUSTER}|g" \ + -e "s|\${LOCATION}|${RC_LOCATION}|g" \ + -e "s|\${MACHINE_TYPE}|${MACHINE_TYPE}|g" \ + -e "s|\${LOADGEN_CPU}|${LOADGEN_CPU}|g" \ + -e "s|\${LOADGEN_MEMORY}|${LOADGEN_MEMORY}|g" \ + -e "s|\${DEADLINE}|${deadline}|g" \ + -e "s|\${ROLE_KEY}|${RC_ROLE_KEY}|g" \ + "${RC_DIR}/manifests/job.yaml.tmpl" > "${dir}/job.yaml" + + # A placeholder added to the template but not to the sed list above would + # otherwise be applied verbatim — a run labelled with something it did not + # do. + local unrendered + # shellcheck disable=SC2016 # literal ${VAR} placeholders are the search target + unrendered="$(grep -o '\${[A-Z_]\+}' "${dir}/job.yaml" | sort -u | tr '\n' ' ')" + if [[ -n "${unrendered}" ]]; then + rc::warn "arm ${arm}c: job.yaml still contains ${unrendered}— add it to the substitution list in run.sh" + return 4 + fi + + rc::kubectl apply -f "${dir}/job.yaml" >/dev/null + + if ! wait_pod_started "${job}"; then + rc::kubectl -n "${RC_JOB_NS}" describe job "${job}" >"${dir}/job.describe" 2>&1 || true + rc::warn "arm ${arm}c: pod never started; see ${dir}/job.describe" + rc::kubectl -n "${RC_JOB_NS}" delete job "${job}" --wait=false >/dev/null 2>&1 || true + return 4 + fi + + # Per-thread CPU sampler, backgrounded for the arm's whole run. Best-effort: + # the arm is complete without it, so its failures go only to threads.err. + local threads_pid="" + "${RC_DIR}/threads.sh" "${pod}" "${RC_ROUTER_NS}" 5 \ + > "${dir}/threads.log" 2> "${dir}/threads.err" & + threads_pid=$! + + # Streamed, not collected at the end: nothing can read files back out of the + # distroless container, and streaming keeps every line an interrupted arm + # already emitted. + # errexit is saved and restored, never forced on: errexit would turn a + # rig-limited arm into killing the whole sweep. + local errexit_was="off" + [[ $- == *e* ]] && errexit_was="on" + set +o errexit + rc::kubectl -n "${RC_JOB_NS}" logs -f "job/${job}" --tail=-1 \ + | python3 "${RC_DIR}/demux.py" "${dir}" + [[ "${errexit_was}" == "on" ]] && set -o errexit + + # Stop the sampler and reduce its stream to per-worker cores. + if [[ -n "${threads_pid}" ]]; then + kill "${threads_pid}" >/dev/null 2>&1 || true + wait "${threads_pid}" 2>/dev/null || true + python3 "${RC_DIR}/threads.py" "${dir}/threads.log" > "${dir}/worker-cpu.jsonl" 2>>"${dir}/threads.err" || true + fi + + local code + code="$(job_exit_code "${job}")" + rc::kubectl -n "${RC_JOB_NS}" delete job "${job}" --cascade=foreground --wait=true >/dev/null 2>&1 || true + + # A clean arm keeps only what downstream reads: samples.jsonl, run.json and + # worker-cpu.jsonl. Debugging material survives only when the arm did not + # exit clean, which is exactly when someone will want it. + if [[ "${code}" -eq 0 ]]; then + rm -f "${dir}/job.yaml" "${dir}/job.log" "${dir}/threads.log" "${dir}/threads.err" + fi + return "${code}" +} + +# --- the sweep --------------------------------------------------------------- + +# Ctrl-C deletes the Job with a foreground cascade, which SIGTERMs the +# generator, which suspends and deletes every actor on the way out. Without +# this a hundred actors survive the run that created them. +# shellcheck disable=SC2317,SC2329 # invoked via the trap below, not by call +cleanup() { + local jobs="" + jobs="$(rc::kubectl -n "${RC_JOB_NS}" get jobs -l app=routercap -o name 2>/dev/null || true)" + if [[ -n "${jobs}" ]]; then + rc::warn "interrupted; deleting ${jobs}" + # shellcheck disable=SC2086 + rc::kubectl -n "${RC_JOB_NS}" delete ${jobs} --cascade=foreground --wait=true >/dev/null 2>&1 || true + fi + exit 2 +} +trap cleanup INT TERM + +rc::step "sweep: arms [${ARMS}] · ${RUNGS} rungs · ${HOLD_S}s each · ${ACTORS} actors · tag ${TAG}" +rc::step "output: ${OUTPUT_DIR}" + +worst=0 +for arm in ${ARMS}; do + set +o errexit + run_arm "${arm}" + code=$? + set -o errexit + case "${code}" in + 0) rc::step "arm ${arm}c: complete" ;; + 2) rc::warn "arm ${arm}c: interrupted"; exit 2 ;; + 3) rc::warn "arm ${arm}c: RIG-LIMITED — the rig ran out, not the router; this arm's numbers are not about the router" ;; + *) rc::warn "arm ${arm}c: failed (exit ${code})" ;; + esac + # The sweep continues past a failed arm. Two good arms and one bad one is a + # partial answer; two unrun arms is none. + (( code > worst )) && worst="${code}" +done + +trap - INT TERM + +# --- charts ------------------------------------------------------------------ + +if [[ "${SKIP_CHARTS}" != "true" ]]; then + rc::step "rendering charts" + set +o errexit + python3 "${RC_DIR}/charts.py" "${OUTPUT_DIR}" + charts_code=$? + set -o errexit + if (( charts_code != 0 )); then + # charts.py reads only the run directory, so a charting failure costs a + # rerun of charts.py, not of the sweep. + rc::warn "charts.py failed (exit ${charts_code}); samples are intact, re-run: python3 ${RC_DIR}/charts.py ${OUTPUT_DIR}" + fi +fi + +rc::step "done: ${OUTPUT_DIR}" +if (( worst != 0 )); then + rc::warn "at least one arm did not finish clean (worst exit ${worst})" +fi +exit "${worst}" diff --git a/benchmarking/routercap/threads.py b/benchmarking/routercap/threads.py new file mode 100644 index 000000000..16f0bbdc6 --- /dev/null +++ b/benchmarking/routercap/threads.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Turns threads.sh's /proc stat stream into per-worker CPU, one JSON line per +sampling interval. + +Input: blocks of "T " followed by /proc//task//stat lines. +Output (stdout): {"t0": .., "t1": .., "workers": {"0": cores, ...}, +"other_cores": .., "max_worker": .., "mean_worker": ..} — workers keyed by the +index in the thread's "wrk:worker_N" name, every other envoy thread (main, +dns, watchdog) folded into other_cores. + +Cores are (utime+stime deltas in jiffies) / USER_HZ / elapsed. USER_HZ is 100 +on every kernel GKE ships; if that ever changes the absolute level shifts but +the skew between workers — the number this file exists for — does not. +""" + +import json +import re +import sys + +USER_HZ = 100.0 + +# tid (comm) state ... utime is field 14, stime 15, 1-indexed after the comm. +# comm can contain spaces but never a ')' for the threads envoy names. +STAT = re.compile(r"^(\d+) \((.*)\) \S (?:\S+ ){10}(\d+) (\d+) ") + + +def parse_blocks(lines): + """Yields (epoch, {tid: (comm, jiffies)}) per sample block.""" + t, threads = None, {} + for line in lines: + if line.startswith("T "): + if t is not None and threads: + yield t, threads + try: + t = int(line.split()[1]) + except (IndexError, ValueError): + t = None + threads = {} + continue + m = STAT.match(line) + if m: + tid, comm, ut, st = m.group(1), m.group(2), int(m.group(3)), int(m.group(4)) + threads[tid] = (comm, ut + st) + if t is not None and threads: + yield t, threads + + +def main(path): + with open(path) as f: + blocks = list(parse_blocks(f)) + prev_t, prev = None, None + for t, threads in blocks: + if prev is not None and t > prev_t: + dt = t - prev_t + workers, other = {}, 0.0 + for tid, (comm, j) in threads.items(): + pj = prev.get(tid) + # A thread that appeared mid-interval has no baseline; skip it + # this round rather than crediting its lifetime total. + if pj is None or pj[0] != comm: + continue + cores = (j - pj[1]) / USER_HZ / dt + if comm.startswith("wrk:worker_"): + workers[comm[len("wrk:worker_"):]] = round(cores, 4) + else: + other += cores + if workers: + vals = list(workers.values()) + print(json.dumps({ + "t0": prev_t, "t1": t, "workers": workers, + "other_cores": round(other, 4), + "max_worker": max(vals), + "mean_worker": round(sum(vals) / len(vals), 4), + })) + prev_t, prev = t, threads + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: threads.py ") + main(sys.argv[1]) diff --git a/benchmarking/routercap/threads.sh b/benchmarking/routercap/threads.sh new file mode 100755 index 000000000..03015cd9d --- /dev/null +++ b/benchmarking/routercap/threads.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Samples per-thread CPU of the router pod's envoy container to stdout, one +# block per interval (default 5s): a "T " line, then every thread's +# /proc stat line. threads.py turns the stream into per-worker cores. +# +# Per-thread CPU exists nowhere else (cAdvisor and Envoy only export sums), so +# it is read from /proc via an ephemeral debug container sharing envoy's PID +# namespace. Best-effort: run.sh backgrounds it and the arm is complete +# without it. +# +# Usage: threads.sh [interval-seconds] + +set -o errexit -o nounset -o pipefail + +POD="${1:?router pod name}" +NS="${2:?namespace}" +INTERVAL="${3:-5}" + +# The debug container joins the envoy container's PID namespace, but pid 1 is +# only envoy if the runtime set it up that way — so find it by comm instead of +# assuming. Everything below runs inside busybox sh on the node. +# shellcheck disable=SC2016 # the single-quoted script must reach busybox unexpanded +exec kubectl -n "${NS}" debug "${POD}" -q --profile=general \ + --image=busybox:1.36 --target=envoy --attach=true -- sh -c ' +pid="" +for p in /proc/[0-9]*; do + if [ "$(cat "$p/comm" 2>/dev/null)" = "envoy" ]; then pid="${p#/proc/}"; break; fi +done +if [ -z "$pid" ]; then echo "no envoy process visible" >&2; exit 1; fi +while true; do + echo "T $(date +%s)" + cat /proc/"$pid"/task/*/stat 2>/dev/null + sleep '"${INTERVAL}"' +done' diff --git a/cmd/benchmarking/routercap/main.go b/cmd/benchmarking/routercap/main.go new file mode 100644 index 000000000..32dca3fe3 --- /dev/null +++ b/cmd/benchmarking/routercap/main.go @@ -0,0 +1,652 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// routercap measures one arm of the atenet-router capacity sweep: one Envoy +// CPU size, one ladder of offered load, one pair of output streams. Arm +// changes live in run.sh, so this binary needs no write access to the cluster. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net/url" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/agent-substrate/substrate/internal/benchmarking/boomer/glutton" + "github.com/agent-substrate/substrate/internal/benchmarking/routercap" + "k8s.io/client-go/kubernetes" +) + +// Exit codes. Distinct so automation can tell "we could not measure this" from +// "the router fell over" without parsing a log. +const ( + exitOK = 0 + exitFailed = 1 + exitInterrupted = 2 + exitRigLimited = 3 + exitPreflight = 4 +) + +type config struct { + // What is being measured. + arm int + pass int + // expectConcurrency overrides the thread count the startup check demands. + // Zero means the arm: cores and threads are one variable except in the + // diagnostic runs that exist to split them. + expectConcurrency int + + // Where things are. + kubeconfig string + apiEndpoint string + routerNamespace string + routerSelector string + routerPods int + workerNamespace string + workerSelector string + loadgenPod string + loadgenNS string + loadgenNode string + loadgenContainer string + + // The ladder. + ladder routercap.LadderSpec + + // The actor pool. + atespace string + actors int + warmConcurrency int + + // The generator's transport. + maxInFlight int64 + requestTimeout time.Duration + fineInterval time.Duration + drainTimeout time.Duration + tickCap time.Duration + + // Sampling. + pollInterval time.Duration + maxWait time.Duration + + // Facts about the deployment under test, recorded so the run explains + // itself and the design's ordering claim can be checked from the output. + portRange string + circuitBreakerLimit int + extProcMaxRequests int + + // Guards. + guards routercap.GuardConfig + + // Output. + outputDir string + dest string + name string + tag string + recordsToStdout bool + gitSHA string + cluster string + location string + machineType string +} + +func main() { + cfg := parseFlags() + // Logs on stderr, records on stdout. Keeping them apart is what lets run.sh + // treat the pod's stdout as a data stream rather than something to grep. + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + + // After flags, before anything runs: the client-port guard's ceiling comes + // from this pod's own source-port range, which the Job spec widens via + // sysctl. The resolved number is serialized into the run header. + cfg.guards.ResolveClientCeiling(os.ReadFile, slog.Default()) + + code := run(cfg) + os.Exit(code) +} + +func parseFlags() *config { + c := &config{guards: routercap.DefaultGuardConfig()} + + flag.IntVar(&c.arm, "arm", 0, "Envoy container CPU limit in cores for this arm. Stamped on every record, and checked against envoy_server_concurrency at startup so a patch that did not take fails here rather than producing a mislabeled series.") + flag.IntVar(&c.expectConcurrency, "expect-concurrency", 0, "Worker-thread count Envoy is expected to report. Zero means the arm's core count — the normal case, where cores and threads are one variable. Set only by diagnostic runs that decouple them (run.sh RC_CONCURRENCY).") + flag.IntVar(&c.pass, "pass", 1, "Which repeat of this arm. Two passes of the same arm that disagree is itself a finding.") + + flag.StringVar(&c.kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only used when not running in-cluster.") + flag.StringVar(&c.apiEndpoint, "api-endpoint", "dns:///api.ate-system.svc.cluster.local:443", "ateapi gRPC dial target, used to warm the actor pool.") + flag.StringVar(&c.routerNamespace, "router-namespace", "ate-system", "Namespace of the router pod under test.") + flag.StringVar(&c.routerSelector, "router-selector", "app=atenet-router", "Label selector for the router pod. Must match exactly -router-pods running pods: more or fewer means a rollout is in progress and measuring the blend would label two configurations as one.") + flag.IntVar(&c.routerPods, "router-pods", 1, "How many router replicas the selector must resolve to. Above 1, load round-robins across the pod IPs (each actor sticks to one replica), while the Envoy/sidecar metric scrapes and the span breakdown describe the first pod only — the per-pod instruments see 1/N of the traffic and the run header says so.") + flag.StringVar(&c.workerNamespace, "worker-namespace", "benchmark-workloads", "Namespace of the worker pods the actors run on.") + flag.StringVar(&c.workerSelector, "worker-selector", "ate.dev/worker-pool", "Label selector for worker pods. Their CPU is recorded because atunnel now terminates mTLS in the request path.") + flag.StringVar(&c.loadgenPod, "loadgen-pod", os.Getenv("POD_NAME"), "This pod's name, for measuring the load generator itself. Defaults to $POD_NAME.") + flag.StringVar(&c.loadgenNS, "loadgen-namespace", os.Getenv("POD_NAMESPACE"), "This pod's namespace. Defaults to $POD_NAMESPACE.") + flag.StringVar(&c.loadgenNode, "loadgen-node", os.Getenv("NODE_NAME"), "The node this pod runs on. Defaults to $NODE_NAME.") + flag.StringVar(&c.loadgenContainer, "loadgen-container", "routercap", "This container's name, as cAdvisor reports it.") + + flag.Float64Var(&c.ladder.StartQPS, "start-qps", 1000, "First rung's offered rate.") + flag.Float64Var(&c.ladder.StepQPS, "step-qps", 1000, "Rate added by each subsequent rung.") + flag.IntVar(&c.ladder.Rungs, "rungs", 16, "Number of rungs. No early stop: the flat region above saturation is data.") + flag.DurationVar(&c.ladder.Hold, "hold", 45*time.Second, "How long each rung runs.") + flag.DurationVar(&c.ladder.Warmup, "warmup", 10*time.Second, "Leading part of each rung flagged as warmup. Still written: a rung's first seconds are where the connection pool grows.") + + flag.StringVar(&c.atespace, "atespace", "routercap", "Atespace the run's actors live in.") + flag.IntVar(&c.actors, "actors", 100, "Actors to warm, one per worker pod. Sized so the per-worker connection-rate limit never binds before the concurrency limit.") + flag.IntVar(&c.warmConcurrency, "warm-concurrency", 16, "Parallelism for actor setup and teardown.") + + flag.Int64Var(&c.maxInFlight, "max-in-flight", 70000, "Generator's own concurrency cap. Reaching it is a rig failure recorded as shed requests, never a statement about the router; set above the widened source-port budget (the Job spec's ip_local_port_range sysctl, 64,511 ports) so the router's limits bind first.") + flag.DurationVar(&c.requestTimeout, "request-timeout", 30*time.Second, "Per-request timeout. Timeouts count as failures and contribute their full latency to the percentiles.") + flag.DurationVar(&c.fineInterval, "fine-interval", time.Second, "Cadence of the generator-only series in fine.jsonl.") + flag.DurationVar(&c.drainTimeout, "drain-timeout", 30*time.Second, "How long to wait for in-flight requests after the last rung.") + flag.DurationVar(&c.tickCap, "tick-cap", 2*time.Millisecond, "Upper bound on the pacer's sleep, which bounds the dispatch lag the dispatch loop itself can add.") + + flag.DurationVar(&c.pollInterval, "cadvisor-poll", time.Second, "How often to re-fetch cAdvisor while waiting for the anchor's timestamp to advance. Well below the kubelet's ~10s cadence so window boundaries are the kubelet's, not the poller's.") + flag.DurationVar(&c.maxWait, "cadvisor-max-wait", 2*time.Minute, "How long a stuck cAdvisor timestamp is tolerated before the arm fails.") + + flag.StringVar(&c.portRange, "port-range", "", "The router pod's measured net.ipv4.ip_local_port_range as \"low-high\". Read from the live pod by run.sh. Left empty the Linux default is assumed and the header says so.") + flag.IntVar(&c.circuitBreakerLimit, "circuit-breaker-limit", 20000, "The actor cluster's circuit-breaker threshold, for the port-budget series. Must match xds.go.") + flag.IntVar(&c.extProcMaxRequests, "extproc-max-requests", 20000, "The router's --extproc-max-requests, recorded so the ordering against the port budget is auditable.") + + flag.Float64Var(&c.guards.LoadgenCPUUtilization, "guard-loadgen-cpu", c.guards.LoadgenCPUUtilization, "Trip when the generator container exceeds this fraction of its own CPU limit. Zero disables.") + flag.Float64Var(&c.guards.WorkerNewConnsPerSec, "guard-worker-conns-per-sec", c.guards.WorkerNewConnsPerSec, "Trip above this mean new-connection rate per worker pod. Zero disables.") + flag.Float64Var(&c.guards.MinRequestsPerConnection, "guard-min-rq-per-cx", c.guards.MinRequestsPerConnection, "Trip when the generator averages fewer requests per connection than this, meaning keep-alive is not holding. Zero disables.") + flag.IntVar(&c.guards.ClientConnectionCeiling, "guard-client-connections", c.guards.ClientConnectionCeiling, "Trip when the generator holds more connections than this, past its own source-port headroom. Zero disables.") + flag.Float64Var(&c.guards.DispatchLagP95Ms, "guard-dispatch-lag-ms", c.guards.DispatchLagP95Ms, "Trip when the generator falls this far behind its own schedule at p95, unless the system is demonstrably saturated. Zero disables.") + flag.Float64Var(&c.guards.SaturationLatencyP95Ms, "saturation-latency-p95-ms", c.guards.SaturationLatencyP95Ms, "p95 latency at or above which the system counts as saturated, suspending the dispatch-lag guard.") + flag.Float64Var(&c.guards.SaturationAchievedRatio, "saturation-achieved-ratio", c.guards.SaturationAchievedRatio, "Achieved-over-offered ratio below which the system counts as saturated.") + + flag.StringVar(&c.outputDir, "output-dir", "", "Directory for samples.jsonl, fine.jsonl and run.json. Overrides --dest.") + flag.StringVar(&c.dest, "dest", "", "Root directory for results; the run lands in ///arm-c. Local paths only.") + flag.StringVar(&c.name, "name", "routercap", "Test name, for the output path and the header.") + flag.StringVar(&c.tag, "tag", "", "Run tag, for the output path and the header.") + flag.BoolVar(&c.recordsToStdout, "records-to-stdout", false, "Also write every record and the header to stdout as tagged JSONL. The router and generator images are distroless, so kubectl cp cannot retrieve a Job's files; this is how an in-cluster run's output gets out. Logs go to stderr either way.") + flag.StringVar(&c.gitSHA, "git-sha", "", "Commit the router image was built from.") + flag.StringVar(&c.cluster, "cluster", "", "Cluster name, recorded in the header.") + flag.StringVar(&c.location, "location", "", "Cluster location, recorded in the header.") + flag.StringVar(&c.machineType, "machine-type", "", "Machine type of the router node. A run on 88-core nodes is not the same experiment as one on 176-core nodes.") + + flag.Parse() + return c +} + +// run is main's body, returning an exit code rather than calling os.Exit, so +// every deferred teardown actually runs. +func run(cfg *config) int { + log := slog.Default() + + outDir, err := cfg.resolveOutputDir() + if err != nil { + log.Error("preflight failed", "err", err) + return exitPreflight + } + + // SIGTERM is how the Job is deleted and Ctrl-C is how a laptop run ends; + // both must unwind through the actor teardown rather than abandon a + // hundred running actors on the worker pods. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + h := routercap.RunHeader{ + Name: cfg.name, + Tag: cfg.tag, + StartedAt: time.Now(), + GitSHA: cfg.gitSHA, + Cluster: cfg.cluster, + Location: cfg.location, + MachineType: cfg.machineType, + CircuitBreakerLimit: cfg.circuitBreakerLimit, + ExtProcMaxRequests: cfg.extProcMaxRequests, + ArmCores: []int{cfg.arm}, + Actors: cfg.actors, + Ladder: cfg.ladder, + Guards: cfg.guards, + Caveats: routercap.StandingCaveats(), + } + var stream *routercap.StreamSink + if cfg.recordsToStdout { + stream = routercap.NewStreamSink(os.Stdout) + } + // Written on every exit path, including the failing ones. A run directory + // whose header is missing because the arm aborted is a directory nobody can + // interpret later. + defer func() { + h.FinishedAt = time.Now() + h.Guards = cfg.guards + if outDir != "" { + if err := routercap.WriteHeader(outDir, h); err != nil { + log.Error("write run header", "err", err) + } + } + if stream != nil { + if err := stream.Header(h); err != nil { + log.Error("stream run header", "err", err) + } + } + }() + + rig, err := setup(ctx, cfg, log) + if rig != nil { + defer rig.close(log) + h.RouterPod = rig.router + if len(rig.routers) > 1 { + h.RouterPods = rig.routers + } + h.Placement = rig.placement + h.RouterImage = rig.router.Images["envoy"] + h.PortRange = rig.portRange + h.Guards = cfg.guards + } + if err != nil { + log.Error("preflight failed", "err", err) + return exitPreflight + } + + var files *routercap.JSONLSink + sinks := routercap.MultiSink{} + if outDir != "" { + files, err = routercap.OpenJSONLSink(outDir) + if err != nil { + log.Error("preflight failed", "err", err) + return exitPreflight + } + defer files.Close() + sinks = append(sinks, files) + } + if stream != nil { + sinks = append(sinks, stream) + } + + runner := &routercap.Runner{ + Arm: cfg.arm, + Pass: cfg.pass, + Rungs: cfg.ladder.Build(), + Client: rig.sender, + Sink: sinks, + Windows: rig.windows, + Envoy: rig.envoy, + Contention: rig.contention, + Router: rig.routerStats, + Targets: rig.targets, + Guards: cfg.guards, + PortRange: rig.portRange, + CircuitBreakerLimit: cfg.circuitBreakerLimit, + MaxInFlight: cfg.maxInFlight, + TickCap: cfg.tickCap, + FineInterval: cfg.fineInterval, + DrainTimeout: cfg.drainTimeout, + Log: log, + } + + log.Info("arm start", + "arm", cfg.arm, "pass", cfg.pass, "rungs", cfg.ladder.Rungs, + "peak_qps", cfg.ladder.PeakQPS(), "actors", cfg.actors, "output", outDir) + + res, runErr := runner.Run(ctx) + h.Results = []routercap.RunResult{res} + + if files != nil { + if err := files.Close(); err != nil { + log.Error("close output files", "err", err) + } + } + + var rigErr *routercap.RigLimitedError + switch { + case errors.As(runErr, &rigErr): + log.Error("arm was rig-limited", "arm", cfg.arm, "err", runErr) + return exitRigLimited + case errors.Is(runErr, context.Canceled): + log.Warn("arm interrupted", "arm", cfg.arm, "windows", res.Windows) + return exitInterrupted + case runErr != nil: + log.Error("arm failed", "arm", cfg.arm, "err", runErr) + return exitFailed + } + + log.Info("arm complete", + "arm", cfg.arm, "pass", cfg.pass, "windows", res.Windows, + "fine_samples", res.FineSamples, "drained", res.Drained, + "clock_skew_ms", res.ClockSkewMs) + return exitOK +} + +// resolveOutputDir picks where the run writes. +func (c *config) resolveOutputDir() (string, error) { + if c.outputDir != "" { + return c.outputDir, nil + } + if c.dest == "" { + // Streaming to stdout is a complete output on its own: it is how the + // in-cluster Job reports. + if c.recordsToStdout { + return "", nil + } + return "", fmt.Errorf("one of --output-dir, --dest or --records-to-stdout is required") + } + // A remote --dest honored by writing somewhere local would lose the run; + // refuse until upload is actually wired in. + if u, err := url.Parse(c.dest); err == nil && u.Scheme != "" { + return "", fmt.Errorf("--dest %q is remote; remote upload is not wired yet, pass a local --output-dir", c.dest) + } + tag := c.tag + if tag == "" { + tag = "untagged" + } + return filepath.Join(c.dest, c.name, tag, fmt.Sprintf("arm-%dc", c.arm)), nil +} + +// rig is everything the runner needs from the cluster, resolved once. +type rig struct { + // router is the anchor pod; routers is every replica, in name order. + // Identical single-element views of the same pod at -router-pods=1. + router routercap.PodRef + routers []routercap.PodRef + placement map[string]string + portRange routercap.PortRange + targets []routercap.Target + windows *routercap.WindowDriver + envoy *routercap.EnvoyClient + contention *routercap.ContentionClient + routerStats *routercap.RouterClient + sender *routercap.Sender + pool *routercap.ActorPool + closers []func() error +} + +// close tears the rig down. Actors first and always: actors left running hold +// worker pods the next arm needs, and the run exits through a signal far more +// often than through a clean finish. +func (r *rig) close(log *slog.Logger) { + if r.pool != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + r.pool.Teardown(ctx) + } + if r.sender != nil { + r.sender.CloseIdleConnections() + } + for _, c := range r.closers { + if err := c(); err != nil { + log.Warn("close", "err", err) + } + } +} + +// setup resolves every source and warms the actor pool. It returns a partially +// built rig even on failure so the caller can still tear down what was created +// and still write a header naming what it found. +func setup(ctx context.Context, cfg *config, log *slog.Logger) (*rig, error) { + r := &rig{placement: map[string]string{}} + + cs, _, err := routercap.NewKubeClient(cfg.kubeconfig) + if err != nil { + return r, err + } + + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + pods := cfg.routerPods + if pods < 1 { + pods = 1 + } + routers, err := routercap.WaitForPods(waitCtx, cs, cfg.routerNamespace, cfg.routerSelector, 2*time.Second, pods) + if err != nil { + return r, fmt.Errorf("resolve router pods: %w", err) + } + // The first pod (by name) anchors everything single-valued: the cAdvisor + // window clock, the Envoy/sidecar scrapes, the port-range read. With + // replicas those instruments see 1/N of the traffic; the generator-side + // series sees all of it. + router := routers[0] + r.router = router + r.routers = routers + r.placement[routercap.RoleEnvoy] = router.Node + for _, p := range routers { + log.Info("router pod", "pod", p.Name, "ip", p.IP, "node", p.Node) + } + + r.portRange, err = parsePortRange(cfg.portRange) + if err != nil { + return r, err + } + + targets, nodes, err := resolveTargets(ctx, cs, cfg, routers, r.placement) + if err != nil { + return r, err + } + r.targets = targets + + // The generator's own container is the one target whose absence would + // disable the guard that matters most, so it is checked here. + if !hasRole(targets, routercap.RoleLoadgen) { + return r, fmt.Errorf("cannot identify this pod's own container: set --loadgen-pod/--loadgen-namespace/--loadgen-node or the POD_NAME/POD_NAMESPACE/NODE_NAME downward-API env vars, or the guard that measures the load generator cannot run") + } + + r.windows = &routercap.WindowDriver{ + Client: routercap.NewMultiNodeCadvisorClient(cs, nodes), + Anchor: router.Key(routercap.RoleEnvoy), + PollInterval: cfg.pollInterval, + MaxWait: cfg.maxWait, + } + + hc := routercap.NewScrapeHTTPClient() + r.envoy = routercap.NewEnvoyClient(hc, router) + r.contention = routercap.NewContentionClient(hc, router) + r.routerStats = routercap.NewRouterClient(hc, router) + + want := cfg.expectConcurrency + if want == 0 { + want = cfg.arm + } + // Every replica, not just the anchor: a rollout that half-took would leave + // one pod on the old thread count, and the blend would be mislabeled. + for _, p := range routers { + if err := checkConcurrency(ctx, routercap.NewEnvoyClient(hc, p), want); err != nil { + return r, fmt.Errorf("pod %s: %w", p.Name, err) + } + } + + conn, api, err := glutton.DialControl(cfg.apiEndpoint, false) + if err != nil { + return r, fmt.Errorf("dial ateapi: %w", err) + } + r.closers = append(r.closers, conn.Close) + + r.pool = &routercap.ActorPool{ + API: api, + Atespace: cfg.atespace, + Concurrency: cfg.warmConcurrency, + Log: log, + } + if err := r.pool.Warm(ctx, cfg.actors); err != nil { + return r, err + } + + urls := make([]string, len(routers)) + for i, p := range routers { + urls[i] = fmt.Sprintf("http://%s:8080", p.IP) + } + r.sender, err = routercap.NewSender(routercap.SenderConfig{ + RouterURLs: urls, + Actors: r.pool.Actors(), + // The idle pool must hold the run's peak concurrency, or Go's idle + // eviction churns connections at exactly the worst load. + MaxConnections: int(cfg.maxInFlight), + RequestTimeout: cfg.requestTimeout, + }) + if err != nil { + return r, err + } + return r, nil +} + +// resolveTargets lists every container the sampler watches and every node whose +// cAdvisor has to be scraped to see them. +func resolveTargets(ctx context.Context, cs kubernetes.Interface, cfg *config, routers []routercap.PodRef, placement map[string]string) ([]routercap.Target, []string, error) { + var targets []routercap.Target + var nodes []string + + // Every replica's envoy and sidecar are watched, so a multi-replica run's + // CPU and memory series sum the whole tier rather than sampling one pod of + // it. Container keys carry the pod name, so same-named containers across + // replicas stay distinct. + for _, router := range routers { + nodes = append(nodes, router.Node) + for _, c := range router.Containers { + role := c + if c != routercap.RoleEnvoy && c != routercap.RoleSidecar { + continue + } + targets = append(targets, routercap.Target{Role: role, Key: router.Key(c)}) + } + } + + if cfg.loadgenPod != "" && cfg.loadgenNS != "" && cfg.loadgenNode != "" { + targets = append(targets, routercap.Target{ + Role: routercap.RoleLoadgen, + Key: routercap.ContainerKey{ + Namespace: cfg.loadgenNS, Pod: cfg.loadgenPod, Container: cfg.loadgenContainer, + }, + }) + nodes = append(nodes, cfg.loadgenNode) + placement[routercap.RoleLoadgen] = cfg.loadgenNode + } + + // Every other pod in the router's namespace is the control plane, listed + // rather than named so new ate-system pods stay covered. A throttled + // ate-api-server is indistinguishable from a slow router at the client. + cp, err := routercap.FindPods(ctx, cs, cfg.routerNamespace, "") + if err != nil { + return nil, nil, fmt.Errorf("list control-plane pods: %w", err) + } + isRouter := map[string]bool{} + for _, router := range routers { + isRouter[router.Name] = true + } + for _, p := range cp { + if isRouter[p.Name] { + continue + } + for _, c := range p.Containers { + targets = append(targets, routercap.Target{Role: routercap.RoleControlPlane, Key: p.Key(c)}) + } + nodes = append(nodes, p.Node) + notePlacement(placement, routercap.RoleControlPlane, p.Node) + } + + workers, err := routercap.FindPods(ctx, cs, cfg.workerNamespace, cfg.workerSelector) + if err != nil { + return nil, nil, fmt.Errorf("list worker pods: %w", err) + } + for _, p := range workers { + for _, c := range p.Containers { + targets = append(targets, routercap.Target{Role: routercap.RoleWorker, Key: p.Key(c)}) + } + nodes = append(nodes, p.Node) + notePlacement(placement, routercap.RoleWorker, p.Node) + } + // Counted, not configured. The per-worker connection-rate guard divides a + // cluster-wide rate by this number, so a flag that disagreed with the + // cluster would move the threshold without anyone noticing. + if len(workers) > 0 { + cfg.guards.WorkerPods = len(workers) + } + + return targets, nodes, nil +} + +// notePlacement records the nodes a role landed on — what was observed, not +// what was intended. A run where the generator shared the router's node is a +// different experiment. +func notePlacement(placement map[string]string, role, node string) { + if node == "" { + return + } + cur := placement[role] + if cur == "" { + placement[role] = node + return + } + for _, n := range strings.Split(cur, ",") { + if n == node { + return + } + } + placement[role] = cur + "," + node +} + +func hasRole(targets []routercap.Target, role string) bool { + for _, t := range targets { + if t.Role == role { + return true + } + } + return false +} + +// checkConcurrency confirms Envoy's worker-thread count matches what the run +// expects — the arm, unless -expect-concurrency decoupled them. Left unset, +// Envoy sizes threads from the *node's* CPU count, so a 10-core arm on a +// 176-core node would run 176 event loops and measure CFS throttling. +func checkConcurrency(ctx context.Context, c *routercap.EnvoyClient, want int) error { + s, err := c.Scrape(ctx) + if err != nil { + return fmt.Errorf("scrape envoy admin: %w", err) + } + if want <= 0 { + return nil + } + if int(s.Concurrency) != want { + return fmt.Errorf("envoy_server_concurrency is %g but %d was expected: the --concurrency patch did not take, and this arm would be labeled with a thread count it is not running", + s.Concurrency, want) + } + return nil +} + +// parsePortRange reads the router pod's measured ephemeral range. Measured +// rather than assumed because every claim about the port wall is a claim about +// these two numbers. +func parsePortRange(s string) (routercap.PortRange, error) { + if strings.TrimSpace(s) == "" { + return routercap.DefaultPortRange(), nil + } + // Accepts both the sysctl's own tab-separated form and "low-high". + f := strings.FieldsFunc(s, func(r rune) bool { + return r == '-' || r == ' ' || r == '\t' || r == ',' + }) + if len(f) != 2 { + return routercap.PortRange{}, fmt.Errorf("--port-range %q: want two numbers, e.g. 32768-60999", s) + } + low, err := strconv.Atoi(f[0]) + if err != nil { + return routercap.PortRange{}, fmt.Errorf("--port-range %q: %w", s, err) + } + high, err := strconv.Atoi(f[1]) + if err != nil { + return routercap.PortRange{}, fmt.Errorf("--port-range %q: %w", s, err) + } + p := routercap.PortRange{Low: low, High: high, Source: routercap.PortRangeMeasured} + if p.Size() <= 0 { + return routercap.PortRange{}, fmt.Errorf("--port-range %q describes no ports", s) + } + return p, nil +} diff --git a/cmd/benchmarking/routercap/main_test.go b/cmd/benchmarking/routercap/main_test.go new file mode 100644 index 000000000..bb7ed40f2 --- /dev/null +++ b/cmd/benchmarking/routercap/main_test.go @@ -0,0 +1,235 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for flag parsing and for the runner configuration the binary assembles. + +package main + +import ( + "context" + "io" + "strings" + "testing" + + "github.com/agent-substrate/substrate/internal/benchmarking/routercap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestParsePortRange(t *testing.T) { + cases := []struct { + name string + in string + wantLow, high int + wantSource string + wantErr bool + }{ + // The sysctl's own output is tab-separated; accepting it directly means + // run.sh can pass what it read rather than reformatting it and getting + // the reformatting wrong. + {name: "SysctlForm", in: "32768\t60999", wantLow: 32768, high: 60999, wantSource: routercap.PortRangeMeasured}, + {name: "DashForm", in: "32768-60999", wantLow: 32768, high: 60999, wantSource: routercap.PortRangeMeasured}, + {name: "SpaceForm", in: "1024 65535", wantLow: 1024, high: 65535, wantSource: routercap.PortRangeMeasured}, + // Unset must say "assumed", not quietly look like a measurement. + {name: "Empty", in: "", wantLow: 32768, high: 60999, wantSource: routercap.PortRangeAssumed}, + {name: "OneNumber", in: "32768", wantErr: true}, + {name: "Reversed", in: "60999-32768", wantErr: true}, + {name: "NotNumbers", in: "low-high", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := parsePortRange(tc.in) + if tc.wantErr { + if err == nil { + t.Fatalf("parsePortRange(%q) = %+v, want an error", tc.in, got) + } + return + } + if err != nil { + t.Fatalf("parsePortRange(%q): %v", tc.in, err) + } + if got.Low != tc.wantLow || got.High != tc.high || got.Source != tc.wantSource { + t.Errorf("parsePortRange(%q) = %+v, want %d-%d from %q", tc.in, got, tc.wantLow, tc.high, tc.wantSource) + } + }) + } +} + +func TestCheckConcurrencyRefusesAMislabeledArm(t *testing.T) { + // Envoy left to its own devices sizes worker threads from the node's core + // count, so a 10-core arm can run 176 event loops and measure CFS + // throttling. Labeling that series "10 cores" is worse than no series. + const body = "envoy_server_concurrency{} 40\n" + c := &routercap.EnvoyClient{Fetch: func(context.Context) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(body)), nil + }} + + if err := checkConcurrency(context.Background(), c, 40); err != nil { + t.Errorf("matching arm rejected: %v", err) + } + err := checkConcurrency(context.Background(), c, 10) + if err == nil { + t.Fatal("a 10-core arm running 40 Envoy workers was accepted") + } + if !strings.Contains(err.Error(), "40") || !strings.Contains(err.Error(), "10") { + t.Errorf("error %q does not name both the observed and the intended concurrency", err) + } + // --arm unset is the smoke-test path: nothing was patched, so there is + // nothing to disagree with. + if err := checkConcurrency(context.Background(), c, 0); err != nil { + t.Errorf("unset arm rejected: %v", err) + } +} + +func TestResolveOutputDir(t *testing.T) { + t.Run("ExplicitWins", func(t *testing.T) { + c := &config{outputDir: "/tmp/here", dest: "/tmp/root", name: "routercap", tag: "t1", arm: 40} + got, err := c.resolveOutputDir() + if err != nil || got != "/tmp/here" { + t.Errorf("got (%q, %v), want /tmp/here", got, err) + } + }) + t.Run("DestBuildsTheArmPath", func(t *testing.T) { + c := &config{dest: "/tmp/root", name: "routercap", tag: "t1", arm: 40} + got, err := c.resolveOutputDir() + if err != nil || got != "/tmp/root/routercap/t1/arm-40c" { + t.Errorf("got (%q, %v), want /tmp/root/routercap/t1/arm-40c", got, err) + } + }) + t.Run("RemoteDestIsRefusedNotIgnored", func(t *testing.T) { + // Honoring a gs:// --dest by writing somewhere local would lose the + // run. Upload is not wired yet, so say so. + c := &config{dest: "gs://bucket/results", name: "routercap", arm: 40} + if _, err := c.resolveOutputDir(); err == nil { + t.Fatal("a remote --dest was silently accepted") + } + }) + t.Run("NeitherIsAnError", func(t *testing.T) { + if _, err := (&config{}).resolveOutputDir(); err == nil { + t.Fatal("a run with nowhere to write was accepted") + } + }) + t.Run("StdoutIsAWholeOutput", func(t *testing.T) { + // How the in-cluster Job reports: nothing can read files back out of a + // distroless container. + got, err := (&config{recordsToStdout: true}).resolveOutputDir() + if err != nil || got != "" { + t.Errorf("got (%q, %v), want no directory and no error", got, err) + } + }) +} + +func testPod(ns, name, node string, containers ...string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, Name: name, + Labels: map[string]string{"ate.dev/worker-pool": "benchmark-ateom", "app": name}, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"}, + Spec: corev1.PodSpec{NodeName: node}, + } + for _, c := range containers { + p.Spec.Containers = append(p.Spec.Containers, corev1.Container{Name: c}) + } + return p +} + +func TestResolveTargetsCoversEveryNodeTheRunWatches(t *testing.T) { + // A node missing from the scrape list leaves its guard permanently silent, + // which reads exactly like the guard passing. + router := routercap.PodRef{ + Namespace: "ate-system", Name: "atenet-router-7d9", IP: "10.0.0.5", Node: "router-node", + Containers: []string{"envoy", "atenet-router"}, + } + cs := fake.NewSimpleClientset( + testPod("ate-system", "atenet-router-7d9", "router-node", "envoy", "atenet-router"), + testPod("ate-system", "ate-api-server-1", "system-node", "ate-api-server"), + testPod("benchmark-workloads", "worker-1", "worker-node-a", "ateom"), + testPod("benchmark-workloads", "worker-2", "worker-node-b", "ateom"), + ) + cfg := &config{ + routerNamespace: "ate-system", + workerNamespace: "benchmark-workloads", + workerSelector: "ate.dev/worker-pool", + loadgenPod: "routercap-runner-abc", + loadgenNS: "benchmarking", + loadgenNode: "loadgen-node", + loadgenContainer: "routercap", + guards: routercap.DefaultGuardConfig(), + } + placement := map[string]string{} + + targets, nodes, err := resolveTargets(context.Background(), cs, cfg, []routercap.PodRef{router}, placement) + if err != nil { + t.Fatalf("resolveTargets: %v", err) + } + + byRole := map[string]int{} + for _, tg := range targets { + byRole[tg.Role]++ + } + want := map[string]int{ + routercap.RoleEnvoy: 1, + routercap.RoleSidecar: 1, + routercap.RoleLoadgen: 1, + routercap.RoleControlPlane: 1, // the api-server; the router pod itself is excluded + routercap.RoleWorker: 2, + } + for role, n := range want { + if byRole[role] != n { + t.Errorf("role %s has %d targets, want %d", role, byRole[role], n) + } + } + + set := map[string]bool{} + for _, n := range nodes { + set[n] = true + } + for _, n := range []string{"router-node", "system-node", "worker-node-a", "worker-node-b", "loadgen-node"} { + if !set[n] { + t.Errorf("node %s is not scraped, so every container on it is invisible", n) + } + } + + // Counted from the cluster, not configured: the per-worker connection-rate + // guard divides a cluster-wide rate by this, so a stale flag would move the + // threshold without anyone noticing. + if cfg.guards.WorkerPods != 2 { + t.Errorf("guard worker pods = %d, want the 2 that were found", cfg.guards.WorkerPods) + } + if got := placement[routercap.RoleWorker]; got != "worker-node-a,worker-node-b" { + t.Errorf("worker placement = %q, want both nodes recorded", got) + } +} + +func TestResolveTargetsWithoutTheDownwardAPI(t *testing.T) { + // Run outside a pod with no --loadgen-* flags: the loadgen target is + // absent, and setup turns that into a startup failure rather than a + // silently disabled guard. + router := routercap.PodRef{ + Namespace: "ate-system", Name: "r", IP: "10.0.0.5", Node: "router-node", + Containers: []string{"envoy", "atenet-router"}, + } + cs := fake.NewSimpleClientset(testPod("ate-system", "r", "router-node", "envoy", "atenet-router")) + cfg := &config{routerNamespace: "ate-system", workerNamespace: "benchmark-workloads", guards: routercap.DefaultGuardConfig()} + + targets, _, err := resolveTargets(context.Background(), cs, cfg, []routercap.PodRef{router}, map[string]string{}) + if err != nil { + t.Fatalf("resolveTargets: %v", err) + } + if hasRole(targets, routercap.RoleLoadgen) { + t.Error("a loadgen target was invented from nothing") + } +} diff --git a/internal/benchmarking/routercap/actors.go b/internal/benchmarking/routercap/actors.go new file mode 100644 index 000000000..69b31b695 --- /dev/null +++ b/internal/benchmarking/routercap/actors.go @@ -0,0 +1,414 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The pool of ate actors the load is aimed at: creating them, warming every one +// before the ladder starts, and suspending and deleting them afterwards. + +package routercap + +import ( + "context" + "fmt" + "log/slog" + "sync" + "sync/atomic" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Every actor is created and resumed during setup and left running for the +// whole arm, so the ladder measures steady-state routing to a warm actor. A +// cold resume costs seconds and would swamp the router's own microseconds. +const ( + // actorTemplateNamespace and actorTemplateName address the existing glutton + // workload, installed by benchmarking/workloads. + actorTemplateNamespace = "benchmark-workloads" + actorTemplateName = "glutton" + // actorDomain is the suffix of the Host header the router routes on. + actorDomain = "actors.resources.substrate.ate.dev" +) + +// Actor is one warmed actor and the Host header that addresses it. +type Actor struct { + Atespace string `json:"atespace"` + Name string `json:"name"` + Host string `json:"host"` +} + +func (a Actor) ref() *ateapipb.ObjectRef { + return &ateapipb.ObjectRef{Atespace: a.Atespace, Name: a.Name} +} + +// ActorPool creates, warms and tears down the actors the ladder addresses. +type ActorPool struct { + API ateapipb.ControlClient + Atespace string + // Concurrency bounds setup and teardown parallelism; a cold boot is ~4s + // per actor, so serial warming would dominate the run. + Concurrency int + // CallTimeout bounds one lifecycle RPC. A cold boot is the slow one. + CallTimeout time.Duration + Log *slog.Logger + + mu sync.Mutex + actors []Actor +} + +func (p *ActorPool) concurrency() int { + if p.Concurrency > 0 { + return p.Concurrency + } + return 16 +} + +func (p *ActorPool) timeout() time.Duration { + if p.CallTimeout > 0 { + return p.CallTimeout + } + return 60 * time.Second +} + +func (p *ActorPool) log() *slog.Logger { + if p.Log != nil { + return p.Log + } + return slog.Default() +} + +// Actors returns the warmed pool. +func (p *ActorPool) Actors() []Actor { + p.mu.Lock() + defer p.mu.Unlock() + return append([]Actor(nil), p.actors...) +} + +// EnsureAtespace creates the run's atespace, treating AlreadyExists as success +// so a re-run against a half-cleaned cluster repairs rather than fails. +func (p *ActorPool) EnsureAtespace(ctx context.Context) error { + cctx, cancel := context.WithTimeout(ctx, p.timeout()) + defer cancel() + _, err := p.API.CreateAtespace(cctx, &ateapipb.CreateAtespaceRequest{ + Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: p.Atespace}}, + }) + if err != nil && status.Code(err) != codes.AlreadyExists { + return fmt.Errorf("create atespace %q: %w", p.Atespace, err) + } + return nil +} + +// Purge deletes every actor already in the run's atespace: each leftover from +// an earlier arm holds a worker pod, and the pool is sized one actor per pod, +// so n leftovers make the next Warm fail with "no free workers available". +// Undeletable actors are reported, not fatal — Warm's all-or-nothing check +// decides whether enough slots came back. +func (p *ActorPool) Purge(ctx context.Context) error { + var stale []Actor + for token := ""; ; { + cctx, cancel := context.WithTimeout(ctx, p.timeout()) + resp, err := p.API.ListActors(cctx, &ateapipb.ListActorsRequest{ + Atespace: p.Atespace, PageSize: 1000, PageToken: token, + }) + cancel() + if err != nil { + return fmt.Errorf("listing actors in %q: %w", p.Atespace, err) + } + for _, a := range resp.GetActors() { + if name := a.GetMetadata().GetName(); name != "" { + stale = append(stale, Actor{Atespace: p.Atespace, Name: name}) + } + } + if token = resp.GetNextPageToken(); token == "" { + break + } + } + if len(stale) == 0 { + return nil + } + + start := time.Now() + var failed atomic.Int64 + _ = p.forEachAll(ctx, stale, func(ctx context.Context, a Actor) error { + if err := p.suspend(ctx, a); err != nil { + p.log().Warn("purge: suspend failed", "actor", a.Name, "err", err) + } + if err := p.delete(ctx, a); err != nil { + p.log().Warn("purge: delete failed", "actor", a.Name, "err", err) + failed.Add(1) + } + return nil + }) + p.log().Info("purged actors left by an earlier arm", + "found", len(stale), "undeletable", failed.Load(), + "elapsed", time.Since(start).Round(time.Millisecond)) + return nil +} + +// Warm creates n actors, resumes each with Boot set, and leaves every one +// running. All-or-nothing: a partial pool concentrates load onto fewer worker +// pods than the per-worker connection-rate guard was sized for. +func (p *ActorPool) Warm(ctx context.Context, n int) error { + if n <= 0 { + return fmt.Errorf("actor count must be positive, got %d", n) + } + if err := p.EnsureAtespace(ctx); err != nil { + return err + } + if err := p.Purge(ctx); err != nil { + return err + } + + start := time.Now() + actors := make([]Actor, n) + for i := range actors { + name := "rc-" + uuid.NewString() + actors[i] = Actor{ + Atespace: p.Atespace, + Name: name, + Host: name + "." + p.Atespace + "." + actorDomain, + } + } + + var slowest atomic.Int64 + // forEachAll, not forEach: abandoned resumes wedge actors in + // STATUS_RESUMING and cost the NEXT arm their worker slots. In-flight + // resumes finish; the error is still returned and warming stays + // all-or-nothing (see benchmarking/routercap/RESULTS.md). + err := p.forEachAll(ctx, actors, func(ctx context.Context, a Actor) error { + t := time.Now() + if err := p.create(ctx, a); err != nil { + return err + } + if err := p.resume(ctx, a, true); err != nil { + return err + } + d := time.Since(t).Nanoseconds() + for { + hi := slowest.Load() + if d <= hi || slowest.CompareAndSwap(hi, d) { + break + } + } + return nil + }) + if err != nil { + // Anything already created is cleaned up: leaving warm actors behind + // would silently consume worker pods on the next attempt. + p.mu.Lock() + p.actors = actors + p.mu.Unlock() + p.Teardown(context.WithoutCancel(ctx)) + return fmt.Errorf("warming %d actors: %w", n, err) + } + + p.mu.Lock() + p.actors = actors + p.mu.Unlock() + p.log().Info("actor pool warm", + "actors", n, "elapsed", time.Since(start).Round(time.Millisecond), + "slowest_cold_boot", time.Duration(slowest.Load()).Round(time.Millisecond)) + return nil +} + +// Teardown suspends and deletes every actor. Best effort and always attempted: +// actors left running hold worker pods that the next arm needs, and the run +// exits far more often through a signal than through a clean finish. +func (p *ActorPool) Teardown(ctx context.Context) { + actors := p.Actors() + if len(actors) == 0 { + return + } + start := time.Now() + var failed atomic.Int64 + _ = p.forEach(ctx, actors, func(ctx context.Context, a Actor) error { + if err := p.suspend(ctx, a); err != nil { + p.log().Warn("suspend actor failed", "actor", a.Name, "err", err) + failed.Add(1) + } + if err := p.delete(ctx, a); err != nil { + p.log().Warn("delete actor failed", "actor", a.Name, "err", err) + failed.Add(1) + } + return nil + }) + p.mu.Lock() + p.actors = nil + p.mu.Unlock() + p.log().Info("actor pool torn down", + "actors", len(actors), "failures", failed.Load(), "elapsed", time.Since(start).Round(time.Millisecond)) +} + +// forEachAll runs fn over every actor with bounded concurrency and returns the +// first error, letting the rest run to completion. Use it whenever fn starts +// something server-side that outlives the RPC and would strand if abandoned. +func (p *ActorPool) forEachAll(ctx context.Context, actors []Actor, fn func(context.Context, Actor) error) error { + sem := make(chan struct{}, p.concurrency()) + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + + for _, a := range actors { + // Cancellation of the *parent* context still stops new work being + // launched; a peer's failure does not cancel work already launched. + select { + case <-ctx.Done(): + wg.Wait() + mu.Lock() + defer mu.Unlock() + if firstErr != nil { + return firstErr + } + return ctx.Err() + case sem <- struct{}{}: + } + wg.Add(1) + go func(a Actor) { + defer wg.Done() + defer func() { <-sem }() + if err := fn(ctx, a); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + }(a) + } + wg.Wait() + mu.Lock() + defer mu.Unlock() + return firstErr +} + +// forEach runs fn over every actor with bounded concurrency, returning the +// first error and canceling the rest. +func (p *ActorPool) forEach(ctx context.Context, actors []Actor, fn func(context.Context, Actor) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sem := make(chan struct{}, p.concurrency()) + var wg sync.WaitGroup + var mu sync.Mutex + var firstErr error + + for _, a := range actors { + select { + case <-ctx.Done(): + wg.Wait() + mu.Lock() + defer mu.Unlock() + if firstErr != nil { + return firstErr + } + return ctx.Err() + case sem <- struct{}{}: + } + wg.Add(1) + go func(a Actor) { + defer wg.Done() + defer func() { <-sem }() + if err := fn(ctx, a); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + cancel() + } + mu.Unlock() + } + }(a) + } + wg.Wait() + mu.Lock() + defer mu.Unlock() + return firstErr +} + +func (p *ActorPool) create(ctx context.Context, a Actor) error { + cctx, cancel := context.WithTimeout(ctx, p.timeout()) + defer cancel() + _, err := p.API.CreateActor(cctx, &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: a.Atespace, Name: a.Name}, + ActorTemplateNamespace: actorTemplateNamespace, + ActorTemplateName: actorTemplateName, + }, + }) + if err != nil && status.Code(err) != codes.AlreadyExists { + return fmt.Errorf("create actor %s: %w", a.Name, err) + } + return nil +} + +func (p *ActorPool) resume(ctx context.Context, a Actor, boot bool) error { + cctx, cancel := context.WithTimeout(ctx, p.timeout()) + defer cancel() + if _, err := p.API.ResumeActor(cctx, &ateapipb.ResumeActorRequest{Actor: a.ref(), Boot: boot}); err != nil { + return fmt.Errorf("resume actor %s (boot=%v): %w", a.Name, boot, err) + } + return nil +} + +func (p *ActorPool) suspend(ctx context.Context, a Actor) error { + return p.settle(ctx, func(ctx context.Context) error { + _, err := p.API.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: a.ref()}) + return err + }) +} + +func (p *ActorPool) delete(ctx context.Context, a Actor) error { + return p.settle(ctx, func(ctx context.Context) error { + _, err := p.API.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: a.ref()}) + return err + }) +} + +// settleTimeout bounds how long a teardown step waits for an actor to leave a +// transitional status. Sized off the ~3.8s cold resume with room for a queue +// behind it. +const settleTimeout = 90 * time.Second + +// settle runs a teardown RPC, retrying on FailedPrecondition: teardown +// routinely races the router's in-flight ResumeActor backlog, so the refusal +// means "not yet", not "no". NotFound is success. +func (p *ActorPool) settle(ctx context.Context, call func(context.Context) error) error { + deadline := time.Now().Add(settleTimeout) + for attempt := 0; ; attempt++ { + cctx, cancel := context.WithTimeout(ctx, p.timeout()) + err := call(cctx) + cancel() + switch { + case err == nil, status.Code(err) == codes.NotFound: + return nil + case status.Code(err) != codes.FailedPrecondition: + return err + case time.Now().After(deadline): + return fmt.Errorf("after %v still in a transitional status: %w", settleTimeout, err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoff(attempt)): + } + } +} + +// backoff ramps 250ms to 2s, capped so parallel retries do not themselves +// become load on ate-api-server. +func backoff(attempt int) time.Duration { + d := 250 * time.Millisecond << min(attempt, 3) + return min(d, 2*time.Second) +} diff --git a/internal/benchmarking/routercap/actors_test.go b/internal/benchmarking/routercap/actors_test.go new file mode 100644 index 000000000..cff57e0a8 --- /dev/null +++ b/internal/benchmarking/routercap/actors_test.go @@ -0,0 +1,410 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for the actor pool, driven against a fake ate-api-server control plane. + +package routercap + +import ( + "context" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeControl implements only the lifecycle calls the pool makes; the embedded +// interface satisfies the rest and panics if anything unexpected is called. +type fakeControl struct { + ateapipb.ControlClient + + mu sync.Mutex + atespaces []string + created []string + resumed []string + bootFlags []bool + suspended []string + deleted []string + createFail map[string]error + resumeFail map[string]error + // existing is what ListActors reports, i.e. what an earlier arm left behind. + existing []string + // refuseFor makes the first n suspend/delete calls for an actor answer + // FailedPrecondition, standing in for an actor still mid-transition. + refuseFor map[string]int + // resumeDelay holds each ResumeActor until released, so a test can observe + // what happens to calls that are still in flight when a sibling fails. + resumeDelay chan struct{} +} + +func newFakeControl() *fakeControl { + return &fakeControl{ + createFail: map[string]error{}, + resumeFail: map[string]error{}, + refuseFor: map[string]int{}, + } +} + +func (f *fakeControl) ListActors(_ context.Context, in *ateapipb.ListActorsRequest, _ ...grpc.CallOption) (*ateapipb.ListActorsResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []*ateapipb.Actor + for _, n := range f.existing { + out = append(out, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: in.GetAtespace(), Name: n}, + }) + } + return &ateapipb.ListActorsResponse{Actors: out}, nil +} + +// refuse reports whether this call should answer FailedPrecondition, consuming +// one of the actor's remaining refusals. Caller holds f.mu. +func (f *fakeControl) refuse(name string) bool { + if f.refuseFor[name] <= 0 { + return false + } + f.refuseFor[name]-- + return true +} + +func (f *fakeControl) CreateAtespace(_ context.Context, in *ateapipb.CreateAtespaceRequest, _ ...grpc.CallOption) (*ateapipb.Atespace, error) { + f.mu.Lock() + defer f.mu.Unlock() + name := in.GetAtespace().GetMetadata().GetName() + for _, a := range f.atespaces { + if a == name { + return nil, status.Error(codes.AlreadyExists, "atespace exists") + } + } + f.atespaces = append(f.atespaces, name) + return &ateapipb.Atespace{}, nil +} + +func (f *fakeControl) CreateActor(_ context.Context, in *ateapipb.CreateActorRequest, _ ...grpc.CallOption) (*ateapipb.Actor, error) { + f.mu.Lock() + defer f.mu.Unlock() + name := in.GetActor().GetMetadata().GetName() + if err, ok := f.createFail[name]; ok { + return nil, err + } + f.created = append(f.created, name) + return &ateapipb.Actor{}, nil +} + +func (f *fakeControl) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + f.mu.Lock() + gate := f.resumeDelay + f.mu.Unlock() + if gate != nil { + // Mirrors the real hazard: the RPC is what holds the resume open, so a + // cancelled context here is a resume abandoned server-side. + select { + case <-gate: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + f.mu.Lock() + defer f.mu.Unlock() + name := in.GetActor().GetName() + if err, ok := f.resumeFail[name]; ok { + return nil, err + } + f.resumed = append(f.resumed, name) + f.bootFlags = append(f.bootFlags, in.GetBoot()) + return &ateapipb.ResumeActorResponse{}, nil +} + +func (f *fakeControl) SuspendActor(_ context.Context, in *ateapipb.SuspendActorRequest, _ ...grpc.CallOption) (*ateapipb.SuspendActorResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + name := in.GetActor().GetName() + if f.refuse(name) { + return nil, status.Error(codes.FailedPrecondition, "got: STATUS_RESUMING, want STATUS_RUNNING") + } + f.suspended = append(f.suspended, name) + return &ateapipb.SuspendActorResponse{}, nil +} + +func (f *fakeControl) DeleteActor(_ context.Context, in *ateapipb.DeleteActorRequest, _ ...grpc.CallOption) (*ateapipb.Actor, error) { + f.mu.Lock() + defer f.mu.Unlock() + name := in.GetActor().GetName() + if f.refuse(name) { + return nil, status.Error(codes.FailedPrecondition, "not in a deletable status") + } + f.deleted = append(f.deleted, name) + return &ateapipb.Actor{}, nil +} + +func (f *fakeControl) counts() (created, resumed, suspended, deleted int) { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.created), len(f.resumed), len(f.suspended), len(f.deleted) +} + +func newTestPool(api ateapipb.ControlClient) *ActorPool { + return &ActorPool{API: api, Atespace: "routercap", Concurrency: 4} +} + +func TestActorPoolWarmsEveryActorWithBoot(t *testing.T) { + f := newFakeControl() + p := newTestPool(f) + + if err := p.Warm(context.Background(), 8); err != nil { + t.Fatalf("Warm: %v", err) + } + + created, resumed, _, _ := f.counts() + if created != 8 || resumed != 8 { + t.Fatalf("created=%d resumed=%d, want 8 and 8", created, resumed) + } + // Boot must be set on the first resume, or the first ping pays the + // cold-start cost the pre-warm exists to remove. + for i, b := range f.bootFlags { + if !b { + t.Fatalf("resume %d had Boot=false", i) + } + } + + actors := p.Actors() + if len(actors) != 8 { + t.Fatalf("pool has %d actors, want 8", len(actors)) + } + seen := map[string]bool{} + for _, a := range actors { + if seen[a.Name] { + t.Fatalf("duplicate actor name %q: two actors would share one worker pod", a.Name) + } + seen[a.Name] = true + want := a.Name + ".routercap." + actorDomain + if a.Host != want { + t.Errorf("Host = %q, want %q", a.Host, want) + } + } +} + +func TestActorPoolTolerantOfAnExistingAtespace(t *testing.T) { + // A re-run against a half-cleaned cluster must repair, not fail. + f := newFakeControl() + f.atespaces = []string{"routercap"} + p := newTestPool(f) + + if err := p.Warm(context.Background(), 2); err != nil { + t.Fatalf("Warm with an existing atespace: %v", err) + } +} + +func TestActorPoolWarmIsAllOrNothing(t *testing.T) { + // A partial pool concentrates the ladder onto fewer worker pods than the + // per-worker connection guard was sized for. That is a different + // experiment, so it has to fail rather than shrink. + f := newFakeControl() + p := newTestPool(f) + // Actor names are random, so fail the first one to arrive rather than a + // named one. + var once sync.Once + p.API = &failingControl{fakeControl: f, failOn: func(string) error { + var err error + once.Do(func() { err = status.Error(codes.ResourceExhausted, "no worker capacity") }) + return err + }} + + err := p.Warm(context.Background(), 6) + if err == nil { + t.Fatal("Warm succeeded despite an actor failing to come up") + } + if !strings.Contains(err.Error(), "no worker capacity") { + t.Errorf("error = %q, want the underlying cause", err) + } + if got := p.Actors(); len(got) != 0 { + t.Errorf("pool retained %d actors after a failed warm", len(got)) + } + // Whatever did get created must be cleaned up, or the next attempt runs + // against a cluster with orphaned actors holding worker pods. + created, _, _, deleted := f.counts() + if deleted < created { + t.Errorf("created %d actors but deleted only %d after the failure", created, deleted) + } +} + +// failingControl injects a create failure for one actor. +type failingControl struct { + *fakeControl + failOn func(name string) error +} + +func (f *failingControl) CreateActor(ctx context.Context, in *ateapipb.CreateActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) { + if err := f.failOn(in.GetActor().GetMetadata().GetName()); err != nil { + return nil, err + } + return f.fakeControl.CreateActor(ctx, in, opts...) +} + +func TestActorPoolTeardownSuspendsThenDeletes(t *testing.T) { + f := newFakeControl() + p := newTestPool(f) + if err := p.Warm(context.Background(), 5); err != nil { + t.Fatalf("Warm: %v", err) + } + + p.Teardown(context.Background()) + + _, _, suspended, deleted := f.counts() + if suspended != 5 || deleted != 5 { + t.Fatalf("suspended=%d deleted=%d, want 5 and 5", suspended, deleted) + } + if got := p.Actors(); len(got) != 0 { + t.Errorf("pool still holds %d actors after teardown", len(got)) + } + // Idempotent: the run tears down on both the clean path and the signal + // path, and those can overlap. + p.Teardown(context.Background()) + _, _, suspended, deleted = f.counts() + if suspended != 5 || deleted != 5 { + t.Errorf("second teardown issued more calls: suspended=%d deleted=%d", suspended, deleted) + } +} + +func TestActorPoolTeardownContinuesPastFailures(t *testing.T) { + // Best effort: one actor that refuses to suspend must not strand the other + // ninety-nine, which the next arm needs released. + f := newFakeControl() + p := newTestPool(&stubbornControl{fakeControl: f}) + if err := p.Warm(context.Background(), 4); err != nil { + t.Fatalf("Warm: %v", err) + } + + p.Teardown(context.Background()) + + if _, _, _, deleted := f.counts(); deleted != 4 { + t.Errorf("deleted %d actors, want all 4 attempted despite the suspend failures", deleted) + } +} + +type stubbornControl struct{ *fakeControl } + +func (s *stubbornControl) SuspendActor(context.Context, *ateapipb.SuspendActorRequest, ...grpc.CallOption) (*ateapipb.SuspendActorResponse, error) { + return nil, status.Error(codes.Internal, "suspend wedged") +} + +func TestActorPoolRejectsAnEmptyPool(t *testing.T) { + if err := newTestPool(newFakeControl()).Warm(context.Background(), 0); err == nil { + t.Fatal("Warm accepted zero actors") + } +} + +func TestActorPoolWarmLetsInFlightResumesFinishWhenAPeerFails(t *testing.T) { + // An abandoned resume wedges the actor in STATUS_RESUMING and holds its + // worker pod, so canceling a failed resume's siblings converts one lost + // actor into n-1 lost worker slots for the next arm. + f := newFakeControl() + gate := make(chan struct{}) + f.resumeDelay = gate + + p := newTestPool(f) + p.Concurrency = 8 + var once sync.Once + p.API = &failingControl{fakeControl: f, failOn: func(string) error { + var err error + once.Do(func() { err = status.Error(codes.FailedPrecondition, "no free workers available") }) + return err + }} + + done := make(chan error, 1) + go func() { done <- p.Warm(context.Background(), 8) }() + + // Let the one failure land and propagate before releasing the rest, so the + // siblings are unambiguously still in flight when Warm learns it failed. + time.Sleep(50 * time.Millisecond) + close(gate) + + err := <-done + if err == nil { + t.Fatal("Warm succeeded despite an actor failing to come up") + } + if !strings.Contains(err.Error(), "no free workers available") { + t.Errorf("error = %q, want the underlying cause", err) + } + // One create failed; every other actor must have completed its resume + // rather than been abandoned mid-flight. + created, resumed, _, _ := f.counts() + if created != 7 || resumed != 7 { + t.Errorf("created %d and resumed %d, want 7 and 7: a resume was abandoned instead of finished", created, resumed) + } +} + +func TestActorPoolTeardownRetriesThroughATransitionalStatus(t *testing.T) { + // Teardown routinely races the router's in-flight ResumeActor backlog, so + // FailedPrecondition means "not yet". Treating it as final strands the + // actor on its worker pod for the rest of the run. + f := newFakeControl() + p := newTestPool(f) + if err := p.Warm(context.Background(), 3); err != nil { + t.Fatalf("Warm: %v", err) + } + // Every actor refuses its first suspend and its first delete. + f.mu.Lock() + for _, a := range p.Actors() { + f.refuseFor[a.Name] = 2 + } + f.mu.Unlock() + + p.Teardown(context.Background()) + + _, _, suspended, deleted := f.counts() + if suspended != 3 || deleted != 3 { + t.Errorf("suspended %d and deleted %d, want 3 and 3: a refusal was treated as final", suspended, deleted) + } +} + +func TestActorPoolWarmPurgesActorsLeftByAnEarlierArm(t *testing.T) { + // The pool is one actor per worker pod, so n leftovers make the next Warm + // fail with "no free workers available"; the purge must remove them. + f := newFakeControl() + f.existing = []string{"rc-stale-1", "rc-stale-2"} + p := newTestPool(f) + + if err := p.Warm(context.Background(), 3); err != nil { + t.Fatalf("Warm: %v", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + for _, want := range f.existing { + if !slices.Contains(f.deleted, want) { + t.Errorf("stale actor %q survived the purge (deleted: %v)", want, f.deleted) + } + } +} + +func TestActorPoolPurgeIsQuietWhenTheAtespaceIsClean(t *testing.T) { + // The common case, and it must cost nothing: no list of actors to delete + // means no suspend and no delete calls at all. + f := newFakeControl() + p := newTestPool(f) + if err := p.Purge(context.Background()); err != nil { + t.Fatalf("Purge: %v", err) + } + if _, _, suspended, deleted := f.counts(); suspended != 0 || deleted != 0 { + t.Errorf("suspended %d and deleted %d against an empty atespace, want 0 and 0", suspended, deleted) + } +} diff --git a/internal/benchmarking/routercap/cadvisor.go b/internal/benchmarking/routercap/cadvisor.go new file mode 100644 index 000000000..dfb24f690 --- /dev/null +++ b/internal/benchmarking/routercap/cadvisor.go @@ -0,0 +1,308 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Scraping the kubelet's cAdvisor endpoint, and turning two scrapes of a container +// into the CPU and memory it used over the interval between them. + +package routercap + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" +) + +// cAdvisor, reached through the kubelet, is the only source that reports raw +// cumulative counters, CFS accounting and a per-container timestamp together. +// metrics.k8s.io serves pre-averaged rates with one timestamp per pod and no +// CFS data, which loses everything the windowing here depends on. +const ( + metricCPUUsageSeconds = "container_cpu_usage_seconds_total" + metricMemoryWorkingSet = "container_memory_working_set_bytes" + metricMemoryRSS = "container_memory_rss" + metricCFSPeriods = "container_cpu_cfs_periods_total" + metricCFSThrottledPeriods = "container_cpu_cfs_throttled_periods_total" + metricCFSThrottledSeconds = "container_cpu_cfs_throttled_seconds_total" + metricSpecCPUQuota = "container_spec_cpu_quota" + metricSpecCPUPeriod = "container_spec_cpu_period" +) + +var cadvisorMetrics = map[string]bool{ + metricCPUUsageSeconds: true, + metricMemoryWorkingSet: true, + metricMemoryRSS: true, + metricCFSPeriods: true, + metricCFSThrottledPeriods: true, + metricCFSThrottledSeconds: true, + metricSpecCPUQuota: true, + metricSpecCPUPeriod: true, +} + +// ContainerKey identifies one container across scrapes. Pod name is included +// because an arm change restarts the router pod, and silently carrying a +// counter across that restart would show up as a large negative CPU rate. +type ContainerKey struct { + Namespace string `json:"namespace"` + Pod string `json:"pod"` + Container string `json:"container"` +} + +func (k ContainerKey) String() string { + return k.Namespace + "/" + k.Pod + "/" + k.Container +} + +// ContainerSample is one container's cumulative counters at one cAdvisor +// housekeeping instant. +type ContainerSample struct { + Key ContainerKey + // At is cAdvisor's own timestamp for the sample, not the time we fetched + // it. The kubelet refreshes on its own ~10s cadence, so a fetch says + // nothing about when the numbers were true. + At time.Time + + CPUSecondsTotal float64 + MemoryWorkingSetBytes float64 + MemoryRSSBytes float64 + CFSPeriods float64 + CFSThrottledPeriods float64 + CFSThrottledSeconds float64 + + // CPUQuota and CPUPeriod are the cgroup's CFS settings, from which the + // effective core limit is derived. A quota of -1 means unlimited. + CPUQuota float64 + CPUPeriod float64 +} + +// LimitCores is the container's CPU limit in cores, or 0 if it has none. A +// BestEffort or Burstable container reports 0 here, which is itself worth +// seeing: it means the arm's x-axis value was not enforced. +func (s ContainerSample) LimitCores() float64 { + if s.CPUQuota <= 0 || s.CPUPeriod <= 0 { + return 0 + } + return s.CPUQuota / s.CPUPeriod +} + +// CadvisorScrape is one fetch of the kubelet's cAdvisor endpoint. +type CadvisorScrape struct { + // FetchedAt is local wall clock at the moment the response was read. + FetchedAt time.Time + // Containers is keyed by namespace/pod/container. Pod-level and + // node-level cgroup rows are dropped. + Containers map[ContainerKey]ContainerSample +} + +// SkewAgainst reports how far the sample for key lags local wall clock at +// fetch time. It conflates real clock skew between this pod and the node with +// cAdvisor housekeeping age, and cannot separate them — which is exactly why +// the run header records the measured value instead of assuming zero. +func (s CadvisorScrape) SkewAgainst(key ContainerKey) (time.Duration, bool) { + c, ok := s.Containers[key] + if !ok { + return 0, false + } + return s.FetchedAt.Sub(c.At), true +} + +// parseCadvisor reads a kubelet cAdvisor exposition into a scrape. +func parseCadvisor(r io.Reader, fetchedAt time.Time) (CadvisorScrape, error) { + out := CadvisorScrape{FetchedAt: fetchedAt, Containers: map[ContainerKey]ContainerSample{}} + err := scanPromText(r, cadvisorMetrics, func(s promSample) { + name := s.Labels["container"] + // cAdvisor also emits rows for the pod sandbox ("POD") and the + // pod-level cgroup (empty container); both double-count the real + // containers. + if name == "" || name == "POD" { + return + } + key := ContainerKey{ + Namespace: s.Labels["namespace"], + Pod: s.Labels["pod"], + Container: name, + } + cs, ok := out.Containers[key] + if !ok { + cs = ContainerSample{Key: key} + } + if s.TimestampMs != 0 { + at := time.UnixMilli(s.TimestampMs) + // Metric families for one container can carry slightly different + // housekeeping timestamps; keep the newest so the interval is never + // credited to an instant before its data. + if at.After(cs.At) { + cs.At = at + } + } + switch s.Name { + case metricCPUUsageSeconds: + cs.CPUSecondsTotal = s.Value + case metricMemoryWorkingSet: + cs.MemoryWorkingSetBytes = s.Value + case metricMemoryRSS: + cs.MemoryRSSBytes = s.Value + case metricCFSPeriods: + cs.CFSPeriods = s.Value + case metricCFSThrottledPeriods: + cs.CFSThrottledPeriods = s.Value + case metricCFSThrottledSeconds: + cs.CFSThrottledSeconds = s.Value + case metricSpecCPUQuota: + cs.CPUQuota = s.Value + case metricSpecCPUPeriod: + cs.CPUPeriod = s.Value + } + out.Containers[key] = cs + }) + if err != nil { + return CadvisorScrape{}, err + } + return out, nil +} + +// ContainerUsage is a container's resource use over one interval. CPU is a +// rate derived from two cumulative samples; memory is a level read at the end +// of the interval, because a working set has no meaningful rate. +type ContainerUsage struct { + Container string `json:"container"` + // CPUCores is mean cores consumed over the interval: seconds of CPU time + // per second of wall clock. + CPUCores float64 `json:"cpu_cores"` + // CPULimitCores is the cgroup quota, so a reader can see utilization + // against the limit rather than against the node. + CPULimitCores float64 `json:"cpu_limit_cores"` + // CPUUtilization is CPUCores/CPULimitCores, or 0 when unlimited. + CPUUtilization float64 `json:"cpu_utilization"` + + MemoryWorkingSetBytes float64 `json:"memory_working_set_bytes"` + MemoryRSSBytes float64 `json:"memory_rss_bytes"` + + // ThrottledPeriods and ThrottledSeconds are deltas over the interval. Any + // throttling on a control-plane container invalidates the arm. + ThrottledPeriods float64 `json:"throttled_periods"` + Periods float64 `json:"periods"` + ThrottledSeconds float64 `json:"throttled_seconds"` + // ThrottledFraction is ThrottledPeriods/Periods. + ThrottledFraction float64 `json:"throttled_fraction"` +} + +// usageBetween derives an interval's usage from two samples of the same +// container. It errors when the samples cannot describe an interval: a counter +// that went backwards means the container restarted. +func usageBetween(prev, cur ContainerSample) (ContainerUsage, error) { + if prev.Key != cur.Key { + return ContainerUsage{}, fmt.Errorf("samples are for different containers: %s and %s", prev.Key, cur.Key) + } + secs := cur.At.Sub(prev.At).Seconds() + if secs <= 0 { + return ContainerUsage{}, fmt.Errorf("%s: non-advancing cAdvisor timestamps (%s to %s)", cur.Key, prev.At, cur.At) + } + if cur.CPUSecondsTotal < prev.CPUSecondsTotal { + return ContainerUsage{}, fmt.Errorf("%s: cpu counter went backwards (%.3f to %.3f): the container restarted mid-interval", + cur.Key, prev.CPUSecondsTotal, cur.CPUSecondsTotal) + } + + u := ContainerUsage{ + Container: cur.Key.Container, + CPUCores: (cur.CPUSecondsTotal - prev.CPUSecondsTotal) / secs, + CPULimitCores: cur.LimitCores(), + MemoryWorkingSetBytes: cur.MemoryWorkingSetBytes, + MemoryRSSBytes: cur.MemoryRSSBytes, + ThrottledPeriods: cur.CFSThrottledPeriods - prev.CFSThrottledPeriods, + Periods: cur.CFSPeriods - prev.CFSPeriods, + ThrottledSeconds: cur.CFSThrottledSeconds - prev.CFSThrottledSeconds, + } + if u.CPULimitCores > 0 { + u.CPUUtilization = u.CPUCores / u.CPULimitCores + } + if u.Periods > 0 { + u.ThrottledFraction = u.ThrottledPeriods / u.Periods + } + return u, nil +} + +// CadvisorClient fetches and parses the kubelet's cAdvisor endpoint for one +// node. +type CadvisorClient struct { + // Fetch returns the raw exposition. Injected so the parsing and windowing + // logic can be tested without a cluster. + Fetch func(ctx context.Context) (io.ReadCloser, error) +} + +// Scrape fetches and parses one sample set. +func (c *CadvisorClient) Scrape(ctx context.Context) (CadvisorScrape, error) { + rc, err := c.Fetch(ctx) + if err != nil { + return CadvisorScrape{}, fmt.Errorf("fetch cadvisor: %w", err) + } + defer rc.Close() + return parseCadvisor(rc, time.Now()) +} + +// Scraper is the window driver's view of cAdvisor. An interface because the +// run watches containers on four different nodes and one node's kubelet only +// reports its own. +type Scraper interface { + Scrape(ctx context.Context) (CadvisorScrape, error) +} + +// MultiNodeClient merges several nodes' cAdvisor surfaces into one scrape. +// Window boundaries still come from the anchor node's clock and each +// container's rate from its own timestamp pair; the cost — other nodes +// housekeeping on their own schedules — is reported as alignment spread. +type MultiNodeClient struct { + Clients []*CadvisorClient +} + +// Scrape fetches every node concurrently — sequential fetches would widen the +// alignment spread — and merges the results. Any node failing fails the +// scrape: a partial merge would silently blind the control-plane guard. +func (m *MultiNodeClient) Scrape(ctx context.Context) (CadvisorScrape, error) { + if len(m.Clients) == 0 { + return CadvisorScrape{}, fmt.Errorf("no cadvisor clients") + } + if len(m.Clients) == 1 { + return m.Clients[0].Scrape(ctx) + } + + scrapes := make([]CadvisorScrape, len(m.Clients)) + errs := make([]error, len(m.Clients)) + var wg sync.WaitGroup + for i, c := range m.Clients { + wg.Add(1) + go func() { + defer wg.Done() + scrapes[i], errs[i] = c.Scrape(ctx) + }() + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return CadvisorScrape{}, err + } + + out := CadvisorScrape{Containers: map[ContainerKey]ContainerSample{}} + for _, s := range scrapes { + // Latest fetch time of the set, so the reported skew is an upper bound + // across every node. + if s.FetchedAt.After(out.FetchedAt) { + out.FetchedAt = s.FetchedAt + } + for k, v := range s.Containers { + out.Containers[k] = v + } + } + return out, nil +} diff --git a/internal/benchmarking/routercap/cadvisor_test.go b/internal/benchmarking/routercap/cadvisor_test.go new file mode 100644 index 000000000..977a987c7 --- /dev/null +++ b/internal/benchmarking/routercap/cadvisor_test.go @@ -0,0 +1,368 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for cAdvisor parsing, per-container usage deltas, and the window driver's +// wait for the kubelet clock to advance. + +package routercap + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "strings" + "testing" + "time" +) + +// cadvisorFixture renders a kubelet-shaped cAdvisor exposition for the router +// pod's two containers at one housekeeping instant. Shaped after the real +// endpoint: HELP/TYPE preamble, per-sample millisecond timestamps, pod-level +// and sandbox rows interleaved with the real containers. +func cadvisorFixture(at time.Time, envoyCPU, routerCPU float64) string { + ms := at.UnixMilli() + var b strings.Builder + b.WriteString("# HELP container_cpu_usage_seconds_total Cumulative cpu time consumed in seconds.\n") + b.WriteString("# TYPE container_cpu_usage_seconds_total counter\n") + row := func(metric, container string, v float64) { + fmt.Fprintf(&b, "%s{container=\"%s\",id=\"/kubepods/pod-abc\",image=\"img\",name=\"k8s_%s\",namespace=\"ate-system\",pod=\"atenet-router-7d9\"} %g %d\n", + metric, container, container, v, ms) + } + // Pod-level cgroup and sandbox rows: these double-count the containers + // below them and must not reach the series. + fmt.Fprintf(&b, "container_cpu_usage_seconds_total{container=\"\",id=\"/kubepods/pod-abc\",namespace=\"ate-system\",pod=\"atenet-router-7d9\"} %g %d\n", envoyCPU+routerCPU, ms) + fmt.Fprintf(&b, "container_cpu_usage_seconds_total{container=\"POD\",id=\"/kubepods/pod-abc/sandbox\",namespace=\"ate-system\",pod=\"atenet-router-7d9\"} 0.4 %d\n", ms) + + row(metricCPUUsageSeconds, "envoy", envoyCPU) + row(metricCPUUsageSeconds, "atenet-router", routerCPU) + row(metricMemoryWorkingSet, "envoy", 1.5e9) + row(metricMemoryWorkingSet, "atenet-router", 0.5e9) + row(metricMemoryRSS, "envoy", 1.2e9) + row(metricMemoryRSS, "atenet-router", 0.4e9) + row(metricCFSPeriods, "envoy", 1000) + row(metricCFSThrottledPeriods, "envoy", 0) + row(metricCFSThrottledSeconds, "envoy", 0) + row(metricSpecCPUQuota, "envoy", 800000) + row(metricSpecCPUPeriod, "envoy", 100000) + row(metricSpecCPUQuota, "atenet-router", 800000) + row(metricSpecCPUPeriod, "atenet-router", 100000) + // An unrelated pod on the same node, to confirm filtering by key. + fmt.Fprintf(&b, "container_cpu_usage_seconds_total{container=\"glutton\",namespace=\"benchmark-workloads\",pod=\"worker-1\"} 12.5 %d\n", ms) + return b.String() +} + +var ( + envoyKey = ContainerKey{Namespace: "ate-system", Pod: "atenet-router-7d9", Container: "envoy"} + routerKey = ContainerKey{Namespace: "ate-system", Pod: "atenet-router-7d9", Container: "atenet-router"} +) + +func TestParseCadvisor(t *testing.T) { + at := time.UnixMilli(1_800_000_000_000) + got, err := parseCadvisor(strings.NewReader(cadvisorFixture(at, 100.0, 20.0)), at.Add(300*time.Millisecond)) + if err != nil { + t.Fatalf("parseCadvisor: %v", err) + } + + t.Run("DropsPodAndSandboxRows", func(t *testing.T) { + for k := range got.Containers { + if k.Container == "" || k.Container == "POD" { + t.Errorf("kept aggregate row %s; it double-counts the real containers", k) + } + } + if len(got.Containers) != 3 { + t.Errorf("got %d containers, want 3 (envoy, atenet-router, glutton)", len(got.Containers)) + } + }) + + t.Run("ReadsTheContainersWeCareAbout", func(t *testing.T) { + envoy, ok := got.Containers[envoyKey] + if !ok { + t.Fatalf("envoy container missing; have %v", got.Containers) + } + if envoy.CPUSecondsTotal != 100.0 { + t.Errorf("envoy cpu = %v, want 100", envoy.CPUSecondsTotal) + } + if envoy.MemoryWorkingSetBytes != 1.5e9 { + t.Errorf("envoy working set = %v, want 1.5e9", envoy.MemoryWorkingSetBytes) + } + if got := envoy.LimitCores(); got != 8 { + t.Errorf("envoy limit = %v cores, want 8 (quota 800000 / period 100000)", got) + } + }) + + t.Run("UsesCadvisorTimestampNotFetchTime", func(t *testing.T) { + envoy := got.Containers[envoyKey] + if !envoy.At.Equal(at) { + t.Errorf("sample time = %v, want the exposition's own timestamp %v", envoy.At, at) + } + skew, ok := got.SkewAgainst(envoyKey) + if !ok || skew != 300*time.Millisecond { + t.Errorf("skew = %v (ok=%v), want 300ms behind fetch time", skew, ok) + } + }) +} + +func TestParseCadvisorHandlesAwkwardLabels(t *testing.T) { + // Container runtimes put command lines in the `name` label, and those + // contain quotes, braces and commas. A naive split on ',' or '}' mangles + // the labels that follow. + in := `container_cpu_usage_seconds_total{name="k8s_envoy_{\"a\":\"b, c\"}",container="envoy",namespace="ate-system",pod="p1"} 7.5 1800000000000 +container_memory_working_set_bytes{container="envoy",namespace="ate-system",pod="p1"} 1024 1800000000000 +` + got, err := parseCadvisor(strings.NewReader(in), time.UnixMilli(1_800_000_000_000)) + if err != nil { + t.Fatalf("parseCadvisor: %v", err) + } + c, ok := got.Containers[ContainerKey{Namespace: "ate-system", Pod: "p1", Container: "envoy"}] + if !ok { + t.Fatalf("container not found; parser was confused by the quoted label. got %v", got.Containers) + } + if c.CPUSecondsTotal != 7.5 || c.MemoryWorkingSetBytes != 1024 { + t.Errorf("got cpu=%v mem=%v, want 7.5 and 1024", c.CPUSecondsTotal, c.MemoryWorkingSetBytes) + } +} + +func TestUsageBetween(t *testing.T) { + t0 := time.UnixMilli(1_800_000_000_000) + t1 := t0.Add(10 * time.Second) + + prev := ContainerSample{Key: envoyKey, At: t0, CPUSecondsTotal: 100, CFSPeriods: 1000, CFSThrottledPeriods: 10, CPUQuota: 800000, CPUPeriod: 100000} + cur := ContainerSample{Key: envoyKey, At: t1, CPUSecondsTotal: 140, CFSPeriods: 2000, CFSThrottledPeriods: 110, CPUQuota: 800000, CPUPeriod: 100000, MemoryWorkingSetBytes: 2e9} + + t.Run("CPUIsARateNotALevel", func(t *testing.T) { + u, err := usageBetween(prev, cur) + if err != nil { + t.Fatalf("usageBetween: %v", err) + } + // 40 CPU-seconds over 10 wall-clock seconds is 4 cores. + if math.Abs(u.CPUCores-4) > 1e-9 { + t.Errorf("cpu = %v cores, want 4", u.CPUCores) + } + if math.Abs(u.CPUUtilization-0.5) > 1e-9 { + t.Errorf("utilization = %v, want 0.5 of an 8-core limit", u.CPUUtilization) + } + if u.MemoryWorkingSetBytes != 2e9 { + t.Errorf("memory = %v, want the level at t1, not a rate", u.MemoryWorkingSetBytes) + } + if math.Abs(u.ThrottledFraction-0.1) > 1e-9 { + t.Errorf("throttled fraction = %v, want 0.1 (100 of 1000 periods)", u.ThrottledFraction) + } + }) + + t.Run("RestartIsAnErrorNotANegativeRate", func(t *testing.T) { + restarted := cur + restarted.CPUSecondsTotal = 2 + if _, err := usageBetween(prev, restarted); err == nil { + t.Error("a counter that went backwards produced a usage value; it must be reported as a restart") + } + }) + + t.Run("NonAdvancingTimestampIsAnError", func(t *testing.T) { + stale := cur + stale.At = t0 + if _, err := usageBetween(prev, stale); err == nil { + t.Error("identical timestamps produced a usage value; the interval is undefined") + } + }) + + t.Run("MismatchedContainersRejected", func(t *testing.T) { + other := cur + other.Key = routerKey + if _, err := usageBetween(prev, other); err == nil { + t.Error("samples from different containers were combined") + } + }) +} + +// stubFetcher serves a scripted sequence of cAdvisor payloads. If repeatLast +// is set, the final body is served forever instead of the sequence running +// out — which is what a genuinely stuck kubelet does, and keeps a timeout test +// from depending on how many polls fit inside its deadline. +type stubFetcher struct { + bodies []string + repeatLast bool + calls int +} + +func (s *stubFetcher) fetch(context.Context) (io.ReadCloser, error) { + if s.calls >= len(s.bodies) { + if !s.repeatLast || len(s.bodies) == 0 { + return nil, errors.New("stub exhausted") + } + s.calls++ + return io.NopCloser(strings.NewReader(s.bodies[len(s.bodies)-1])), nil + } + b := s.bodies[s.calls] + s.calls++ + return io.NopCloser(strings.NewReader(b)), nil +} + +func TestWindowDriverWaitsForTheKubeletClock(t *testing.T) { + // The core alignment property: a stale scrape must extend the wait, never + // produce a window. Two repeats of the same timestamp, then an advance. + t0 := time.UnixMilli(1_800_000_000_000) + t1 := t0.Add(10 * time.Second) + stub := &stubFetcher{bodies: []string{ + cadvisorFixture(t0, 100, 20), // Prime + cadvisorFixture(t0, 100, 20), // stale + cadvisorFixture(t0, 100, 20), // stale + cadvisorFixture(t1, 140, 25), // kubelet housekept + }} + d := &WindowDriver{ + Client: &CadvisorClient{Fetch: stub.fetch}, + Anchor: envoyKey, + PollInterval: time.Millisecond, + MaxWait: 5 * time.Second, + } + + w, err := d.Next(context.Background()) + if err != nil { + t.Fatalf("Next: %v", err) + } + if !w.T0.Equal(t0) || !w.T1.Equal(t1) { + t.Errorf("window = [%v, %v), want [%v, %v) from cAdvisor's own timestamps", w.T0, w.T1, t0, t1) + } + if w.Polls != 3 { + t.Errorf("polls = %d, want 3: the two stale scrapes must extend the wait", w.Polls) + } + if w.Duration() != 10*time.Second { + t.Errorf("duration = %v, want 10s", w.Duration()) + } + if got := w.Mid(); !got.Equal(t0.Add(5 * time.Second)) { + t.Errorf("mid = %v, want the interval midpoint", got) + } + + usage, spread, missing, errs := w.Usage([]ContainerKey{envoyKey, routerKey}) + if len(missing) != 0 || len(errs) != 0 { + t.Fatalf("usage missing=%v errs=%v", missing, errs) + } + if got := usage[envoyKey].CPUCores; math.Abs(got-4) > 1e-9 { + t.Errorf("envoy cpu = %v cores, want 4 (40 cpu-seconds over 10s)", got) + } + if got := usage[routerKey].CPUCores; math.Abs(got-0.5) > 1e-9 { + t.Errorf("router cpu = %v cores, want 0.5", got) + } + if spread != 0 { + t.Errorf("spread = %v, want 0: both containers share the anchor's interval", spread) + } +} + +func TestWindowDriverTimesOutOnAStuckKubelet(t *testing.T) { + // A kubelet that stops housekeeping must fail loudly. Emitting a record + // anyway would pair a fresh QPS number with a CPU reading from minutes ago + // and look entirely plausible on the chart. + t0 := time.UnixMilli(1_800_000_000_000) + stub := &stubFetcher{bodies: []string{cadvisorFixture(t0, 100, 20)}, repeatLast: true} + d := &WindowDriver{ + Client: &CadvisorClient{Fetch: stub.fetch}, + Anchor: envoyKey, + PollInterval: time.Millisecond, + MaxWait: 50 * time.Millisecond, + } + if _, err := d.Next(context.Background()); err == nil { + t.Fatal("Next returned a window from a stuck kubelet") + } else if !strings.Contains(err.Error(), "stuck") { + t.Errorf("error = %v, want it to name the stuck timestamp", err) + } +} + +func TestWindowDriverReportsAMissingAnchor(t *testing.T) { + // Expected across an arm change: the router pod is replaced, so the anchor + // key no longer resolves and the caller must re-resolve the pod. + t0 := time.UnixMilli(1_800_000_000_000) + stub := &stubFetcher{bodies: []string{cadvisorFixture(t0, 100, 20), `# nothing here`}} + d := &WindowDriver{ + Client: &CadvisorClient{Fetch: stub.fetch}, + Anchor: envoyKey, + PollInterval: time.Millisecond, + MaxWait: time.Second, + } + if err := d.Prime(context.Background()); err != nil { + t.Fatalf("Prime: %v", err) + } + _, err := d.Next(context.Background()) + if !errors.Is(err, ErrAnchorMissing) { + t.Errorf("error = %v, want ErrAnchorMissing so the caller can re-resolve the pod", err) + } +} + +func TestWindowUsageReportsMissingContainersSeparately(t *testing.T) { + // "The router used no CPU" and "we could not see the router" must not + // render as the same point. + t0 := time.UnixMilli(1_800_000_000_000) + t1 := t0.Add(10 * time.Second) + prev, _ := parseCadvisor(strings.NewReader(cadvisorFixture(t0, 100, 20)), t0) + cur, _ := parseCadvisor(strings.NewReader(cadvisorFixture(t1, 140, 25)), t1) + w := Window{T0: t0, T1: t1, Prev: prev, Cur: cur} + + ghost := ContainerKey{Namespace: "ate-system", Pod: "atenet-router-7d9", Container: "not-a-container"} + usage, _, missing, errs := w.Usage([]ContainerKey{envoyKey, ghost}) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(missing) != 1 || missing[0] != ghost { + t.Errorf("missing = %v, want exactly the absent container", missing) + } + if _, present := usage[ghost]; present { + t.Error("absent container got a usage entry") + } + if _, present := usage[envoyKey]; !present { + t.Error("one missing container suppressed the containers that were present") + } +} + +func TestMultiNodeClientMergesEveryNode(t *testing.T) { + // The control-plane throttling guard is only answerable from the system + // node's kubelet, and the router's kubelet reports only the router's node. + // One Window has to span both. + at := time.UnixMilli(1_800_000_000_000) + systemRow := fmt.Sprintf( + "container_cpu_usage_seconds_total{container=\"ate-api-server\",namespace=\"ate-system\",pod=\"ate-api-server-1\"} 55 %d\n", + at.UnixMilli()) + + m := &MultiNodeClient{Clients: []*CadvisorClient{ + {Fetch: (&stubFetcher{bodies: []string{cadvisorFixture(at, 100, 20)}, repeatLast: true}).fetch}, + {Fetch: (&stubFetcher{bodies: []string{systemRow}, repeatLast: true}).fetch}, + }} + + got, err := m.Scrape(context.Background()) + if err != nil { + t.Fatalf("Scrape: %v", err) + } + if _, ok := got.Containers[envoyKey]; !ok { + t.Errorf("router node's containers missing from the merge: %v", got.Containers) + } + apiKey := ContainerKey{Namespace: "ate-system", Pod: "ate-api-server-1", Container: "ate-api-server"} + if _, ok := got.Containers[apiKey]; !ok { + t.Errorf("system node's containers missing from the merge: %v", got.Containers) + } +} + +func TestMultiNodeClientFailsWhenAnyNodeFails(t *testing.T) { + // A partial merge would silently drop whichever node was unreachable, and + // a guard that cannot see the control plane never trips. + at := time.UnixMilli(1_800_000_000_000) + m := &MultiNodeClient{Clients: []*CadvisorClient{ + {Fetch: (&stubFetcher{bodies: []string{cadvisorFixture(at, 100, 20)}, repeatLast: true}).fetch}, + {Fetch: func(context.Context) (io.ReadCloser, error) { return nil, errors.New("kubelet unreachable") }}, + }} + + if _, err := m.Scrape(context.Background()); err == nil { + t.Fatal("Scrape succeeded with one node down") + } else if !strings.Contains(err.Error(), "kubelet unreachable") { + t.Errorf("err = %v, want it to name the failure", err) + } +} diff --git a/internal/benchmarking/routercap/collector.go b/internal/benchmarking/routercap/collector.go new file mode 100644 index 000000000..866fc2b67 --- /dev/null +++ b/internal/benchmarking/routercap/collector.go @@ -0,0 +1,266 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The in-memory tally of every request the generator scheduled — dispatch lag, +// latency, outcome and peak concurrency — sliceable by an arbitrary time window. + +package routercap + +import ( + "sync" + "sync/atomic" + "time" +) + +// Outcome classifies how one request ended. Failures the generator produced +// (shed) mean the rig ran out, not the router, and must never be read as the +// router refusing load. +type Outcome string + +const ( + // OutcomeOK is a 2xx whose body echoed back what was sent. + OutcomeOK Outcome = "ok" + // OutcomeHTTPError is a response with a >=400 status, including the 503s + // Envoy generates when a circuit breaker trips. + OutcomeHTTPError Outcome = "httperror" + // OutcomeTransportError is a connection that never produced a response: + // refused, reset, timed out, or EOF mid-body. + OutcomeTransportError Outcome = "transport" + // OutcomeBadBody is a 2xx whose payload did not round-trip: something + // answered, but not the actor we addressed. + OutcomeBadBody Outcome = "badbody" + // OutcomeShed is a request the generator declined to send because its own + // in-flight cap was reached. Never a statement about the router. + OutcomeShed Outcome = "shed" +) + +// completion is one finished request. Latency is measured from Scheduled, not +// Dispatched, to avoid coordinated omission. +type completion struct { + Scheduled time.Time + Completed time.Time + Outcome Outcome + Status int +} + +// dispatch records that a request reached the wire. Emitted separately from +// its completion so a window's dispatch-lag statistics include requests still +// in flight when the window closes. +type dispatch struct { + Scheduled time.Time + Dispatched time.Time +} + +// peakSlot names one consumer of the in-flight high-water mark. Reading a slot +// resets it, so each consumer needs its own. +type peakSlot int + +const ( + // peakAligned is the ~10s series in samples.jsonl. + peakAligned peakSlot = iota + // peakFine is the 1s generator-only series in fine.jsonl. + peakFine + numPeakSlots +) + +// Collector accumulates raw per-request events and answers interval queries +// over them. Raw events rather than pre-aggregated buckets, because window +// boundaries come from cAdvisor's clock (see window.go) and are not known in +// advance. +type Collector struct { + mu sync.Mutex + completions []completion + dispatches []dispatch + + inFlight atomic.Int64 + // maxInFlight holds one high-water slot per consumer. A single slot shared + // by the ~10s and 1s series would let whichever read last hand the other a + // maximum covering only the sliver since that read. + maxInFlight [numPeakSlots]atomic.Int64 + + // schedule answers "how many requests were due in this interval": the + // pacer's schedule is deterministic, so offered load is arithmetic, not + // measurement. + schedule *Schedule +} + +// NewCollector returns a Collector reporting offered load from sched. +func NewCollector(sched *Schedule) *Collector { + return &Collector{schedule: sched} +} + +// RecordDispatch notes that a request scheduled for sched reached the wire at +// at. It also raises the in-flight count, which RecordCompletion lowers. +func (c *Collector) RecordDispatch(sched, at time.Time) { + n := c.inFlight.Add(1) + for i := range c.maxInFlight { + for { + hi := c.maxInFlight[i].Load() + if n <= hi || c.maxInFlight[i].CompareAndSwap(hi, n) { + break + } + } + } + c.mu.Lock() + c.dispatches = append(c.dispatches, dispatch{Scheduled: sched, Dispatched: at}) + c.mu.Unlock() +} + +// RecordCompletion notes a finished request. Callers must pair it with exactly +// one RecordDispatch, except for OutcomeShed which never reached the wire and +// so is recorded with RecordShed instead. +func (c *Collector) RecordCompletion(sched, at time.Time, outcome Outcome, status int) { + c.inFlight.Add(-1) + c.mu.Lock() + c.completions = append(c.completions, completion{ + Scheduled: sched, Completed: at, Outcome: outcome, Status: status, + }) + c.mu.Unlock() +} + +// RecordShed notes a request the generator refused to send. It is counted but +// excluded from the latency distribution. +func (c *Collector) RecordShed(sched, at time.Time) { + c.mu.Lock() + c.completions = append(c.completions, completion{ + Scheduled: sched, Completed: at, Outcome: OutcomeShed, + }) + c.mu.Unlock() +} + +// InFlight is the number of requests currently sent but unanswered. +func (c *Collector) InFlight() int64 { return c.inFlight.Load() } + +// GenStats is everything the generator knows about one interval. Every field +// is scoped to the same [t0, t1) as the resource samples it will be written +// alongside. +type GenStats struct { + // OfferedQPS is the rate the pacer was scheduled to produce, independent of + // what happened. + OfferedQPS float64 `json:"offered_qps"` + // DispatchedQPS is the rate that actually reached the wire. Below offered + // means the generator itself could not keep up. + DispatchedQPS float64 `json:"dispatched_qps"` + // AchievedQPS is the rate of completed requests, successful or not. Below + // offered means the system did not keep up. + AchievedQPS float64 `json:"achieved_qps"` + // SuccessQPS counts only OutcomeOK. The gap to AchievedQPS is the error + // rate expressed in the same units as the rest of the chart. + SuccessQPS float64 `json:"success_qps"` + + // InFlightEnd is the concurrency at t1, and InFlightMax the high-water + // mark during the window; both are sampled, not integrated. On an HTTP/1.1 + // upstream each in-flight request holds one connection and one ephemeral + // source port. + InFlightEnd int64 `json:"in_flight_end"` + InFlightMax int64 `json:"in_flight_max"` + + // Latency is measured from scheduled send time and includes failures, so + // a timeout raises the tail instead of vanishing from it. + Latency LatencyStats `json:"latency"` + // DispatchLag is scheduled-to-wire delay: the generator measuring itself. + // Large means the generator fell behind and that part of the curve + // describes the rig. + DispatchLag LatencyStats `json:"dispatch_lag"` + + Outcomes map[Outcome]int `json:"outcomes"` + Statuses map[int]int `json:"statuses,omitempty"` +} + +// Stats summarizes the interval [t0, t1). Completions are attributed by +// completion instant and dispatches by dispatch instant, so a slow request +// contributes its lag to one window and its latency to a later one. +func (c *Collector) Stats(t0, t1 time.Time) GenStats { + return c.stats(t0, t1, peakAligned) +} + +// FineStats is Stats for the 1s generator-only series. It differs only in +// which in-flight high-water slot it reads and resets. +func (c *Collector) FineStats(t0, t1 time.Time) GenStats { + return c.stats(t0, t1, peakFine) +} + +func (c *Collector) stats(t0, t1 time.Time, slot peakSlot) GenStats { + secs := t1.Sub(t0).Seconds() + if secs <= 0 { + return GenStats{Outcomes: map[Outcome]int{}} + } + + c.mu.Lock() + var lats []time.Duration + outcomes := map[Outcome]int{} + statuses := map[int]int{} + var completed, success int + for _, cp := range c.completions { + if cp.Completed.Before(t0) || !cp.Completed.Before(t1) { + continue + } + completed++ + outcomes[cp.Outcome]++ + if cp.Status != 0 { + statuses[cp.Status]++ + } + if cp.Outcome == OutcomeOK { + success++ + } + if cp.Outcome != OutcomeShed { + lats = append(lats, cp.Completed.Sub(cp.Scheduled)) + } + } + var lags []time.Duration + var dispatched int + for _, d := range c.dispatches { + if d.Dispatched.Before(t0) || !d.Dispatched.Before(t1) { + continue + } + dispatched++ + lags = append(lags, d.Dispatched.Sub(d.Scheduled)) + } + c.mu.Unlock() + + return GenStats{ + OfferedQPS: c.schedule.OfferedIn(t0, t1) / secs, + DispatchedQPS: float64(dispatched) / secs, + AchievedQPS: float64(completed) / secs, + SuccessQPS: float64(success) / secs, + InFlightEnd: c.inFlight.Load(), + InFlightMax: c.maxInFlight[slot].Swap(c.inFlight.Load()), + Latency: summarize(lats), + DispatchLag: summarize(lags), + Outcomes: outcomes, + Statuses: statuses, + } +} + +// Prune drops events that completed or dispatched before cutoff. Callers must +// keep cutoff at or below the earliest interval they still intend to query — +// with two consumers, the older of their two frontiers. +func (c *Collector) Prune(cutoff time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + keptC := c.completions[:0] + for _, cp := range c.completions { + if !cp.Completed.Before(cutoff) { + keptC = append(keptC, cp) + } + } + c.completions = keptC + keptD := c.dispatches[:0] + for _, d := range c.dispatches { + if !d.Dispatched.Before(cutoff) { + keptD = append(keptD, d) + } + } + c.dispatches = keptD +} diff --git a/internal/benchmarking/routercap/envoy.go b/internal/benchmarking/routercap/envoy.go new file mode 100644 index 000000000..d16ee725c --- /dev/null +++ b/internal/benchmarking/routercap/envoy.go @@ -0,0 +1,719 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Scraping Envoy's admin /stats and the sidecar's /metrics, and deltaing the +// counters, which are cumulative since process start rather than per-interval. + +package routercap + +import ( + "context" + "encoding/json" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" +) + +// Envoy's Prometheus counters are cumulative since process start, so every +// counter in the output series is a difference between two scrapes taken at +// the window boundaries. Gauges are read at the closing scrape. +const ( + // ActorClusterName and ExtProcClusterName must match the xDS cluster names + // in cmd/atenet/internal/router/xds.go. Every request traverses both, so + // either can be the thing that saturates. + ActorClusterName = "actor_original_dst" + ExtProcClusterName = "ate-cluster" +) + +const ( + mClusterCxActive = "envoy_cluster_upstream_cx_active" + mClusterCxTotal = "envoy_cluster_upstream_cx_total" + mClusterCxOverflow = "envoy_cluster_upstream_cx_overflow" + mClusterCxConnectFail = "envoy_cluster_upstream_cx_connect_fail" + mClusterCxConnectTimeout = "envoy_cluster_upstream_cx_connect_timeout" + mClusterRqActive = "envoy_cluster_upstream_rq_active" + mClusterRqTotal = "envoy_cluster_upstream_rq_total" + mClusterRqTimeout = "envoy_cluster_upstream_rq_timeout" + mClusterRqRetry = "envoy_cluster_upstream_rq_retry" + mClusterRqPendingActive = "envoy_cluster_upstream_rq_pending_active" + mClusterRqPendingOverflw = "envoy_cluster_upstream_rq_pending_overflow" + mClusterCbCxOpen = "envoy_cluster_circuit_breakers_default_cx_open" + mClusterCbRqOpen = "envoy_cluster_circuit_breakers_default_rq_open" + mClusterCbPendingOpen = "envoy_cluster_circuit_breakers_default_rq_pending_open" + + // Deliberately no circuit_breakers.default.remaining_* here: without + // track_remaining they read a constant zero, the inverse of the truth. + // Headroom is recoverable as circuit_breaker_limit minus rq_active. + + mServerConcurrency = "envoy_server_concurrency" + mServerMemoryAllocated = "envoy_server_memory_allocated" + mServerMemoryHeapSize = "envoy_server_memory_heap_size" + mServerTotalConns = "envoy_server_total_connections" + + mDownstreamCxActive = "envoy_http_downstream_cx_active" + mDownstreamRqTotal = "envoy_http_downstream_rq_total" + + // Both histograms are read only through _sum and _count for an exact mean; + // a mean is the only statistic that stacks across hops, and Envoy's bucket + // edges are too coarse for percentiles anyway. Envoy publishes these in + // milliseconds, unlike the sidecar's seconds. + mDownstreamRqTimeSum = "envoy_http_downstream_rq_time_sum" + mDownstreamRqTimeCount = "envoy_http_downstream_rq_time_count" + mClusterRqTimeSum = "envoy_cluster_upstream_rq_time_sum" + mClusterRqTimeCount = "envoy_cluster_upstream_rq_time_count" + + // The ext_proc leg runs over exactly one HTTP/2 connection per Envoy worker + // thread, so these gauges read directly on whether that connection is the + // constriction. Emitted for every http2 cluster; only meaningful for + // ate-cluster. + mClusterH2StreamsActive = "envoy_cluster_http2_streams_active" + mClusterH2PendingSend = "envoy_cluster_http2_pending_send_bytes" + + // Per-worker-thread series, all labeled envoy_worker_id; sums across + // threads hide a single pinned worker, which is what these exist to see. + // Watchdog misses are event-loop stalls past 200ms/1s; the dispatcher loop + // histogram appears only with enable_dispatcher_stats: true in the + // bootstrap (fields stay zero otherwise). + mWorkerWatchdogMiss = "envoy_server_worker_watchdog_miss" + mWorkerWatchdogMegaMiss = "envoy_server_worker_watchdog_mega_miss" + mListenerWorkerCx = "envoy_listener_worker_downstream_cx_total" + // listener_manager.worker_.dispatcher.loop_duration_us, as the v1.30 + // admin endpoint actually mangles it (verified live). + mWorkerLoopUsSum = "envoy_listener_manager_worker_dispatcher_loop_duration_us_sum" + mWorkerLoopUsCount = "envoy_listener_manager_worker_dispatcher_loop_duration_us_count" +) + +// adminConnManagerPrefix names Envoy's own admin listener in the +// downstream_rq_time label set. Its requests are this harness scraping /stats +// and are excluded so they do not drag the in-Envoy mean toward zero. +const adminConnManagerPrefix = "admin" + +var envoyMetrics = map[string]bool{ + mClusterCxActive: true, mClusterCxTotal: true, mClusterCxOverflow: true, + mClusterCxConnectFail: true, mClusterCxConnectTimeout: true, + mClusterRqActive: true, mClusterRqTotal: true, mClusterRqTimeout: true, + mClusterRqRetry: true, mClusterRqPendingActive: true, mClusterRqPendingOverflw: true, + mClusterCbCxOpen: true, mClusterCbRqOpen: true, mClusterCbPendingOpen: true, + mServerConcurrency: true, mServerMemoryAllocated: true, mServerMemoryHeapSize: true, + mServerTotalConns: true, mDownstreamCxActive: true, mDownstreamRqTotal: true, + mDownstreamRqTimeSum: true, mDownstreamRqTimeCount: true, + mClusterRqTimeSum: true, mClusterRqTimeCount: true, + mClusterH2StreamsActive: true, mClusterH2PendingSend: true, + mWorkerWatchdogMiss: true, mWorkerWatchdogMegaMiss: true, mListenerWorkerCx: true, + mWorkerLoopUsSum: true, mWorkerLoopUsCount: true, +} + +// WorkerCounters is one Envoy worker thread's series at one instant, keyed off +// the envoy_worker_id label. +type WorkerCounters struct { + WatchdogMiss float64 // counter + WatchdogMegaMiss float64 // counter + AcceptedCx float64 // counter, summed across the non-admin listeners + LoopDurUsSum float64 // histogram sum, microseconds + LoopDurUsCount float64 // histogram count +} + +// ClusterStats is one Envoy cluster's counters and gauges at one instant. +type ClusterStats struct { + // Gauges. + CxActive float64 + RqActive float64 + RqPendingActive float64 + CbCxOpen float64 + CbRqOpen float64 + CbPendingOpen float64 + + // Counters, cumulative since process start. + CxTotal float64 + CxOverflow float64 + CxConnectFail float64 + CxConnectTimeout float64 + RqTotal float64 + RqTimeout float64 + RqRetry float64 + RqPendingOverflow float64 + + // RqTimeMsTotal and RqTimeCount are the upstream_rq_time histogram's sum and + // count. Only the actor cluster has them: ext_proc streams all end in a + // reset, so Envoy never records a completion time for the ate-cluster. + RqTimeMsTotal float64 + RqTimeCount float64 + + // Gauges. See mClusterH2StreamsActive for why these exist. + H2StreamsActive float64 + H2PendingSendBytes float64 +} + +// EnvoyStats is one scrape of Envoy's admin Prometheus endpoint. +type EnvoyStats struct { + At time.Time + + // Concurrency is the number of worker threads Envoy started, checked + // against the arm's CPU limit. Unset, Envoy sizes this from the node's + // core count and the arm would measure CFS throttling instead of the proxy. + Concurrency float64 + MemoryAllocated float64 + MemoryHeapSize float64 + TotalConnections float64 + + DownstreamCxActive float64 + DownstreamRqTotal float64 + + // DownstreamRqTimeMsTotal and DownstreamRqTimeCount are the whole time a + // request spent inside Envoy, from request headers received to response + // complete. Every other hop is carved out of this one. + DownstreamRqTimeMsTotal float64 + DownstreamRqTimeCount float64 + + Clusters map[string]ClusterStats + // Workers is keyed by the envoy_worker_id label ("0" .. concurrency-1). + Workers map[string]WorkerCounters +} + +func parseEnvoyStats(r io.Reader, at time.Time) (EnvoyStats, error) { + out := EnvoyStats{At: at, Clusters: map[string]ClusterStats{}, Workers: map[string]WorkerCounters{}} + worker := func(s promSample, apply func(*WorkerCounters)) { + id := s.Labels["envoy_worker_id"] + if id == "" { + return + } + w := out.Workers[id] + apply(&w) + out.Workers[id] = w + } + err := scanPromText(r, envoyMetrics, func(s promSample) { + switch s.Name { + case mWorkerWatchdogMiss: + worker(s, func(w *WorkerCounters) { w.WatchdogMiss += s.Value }) + return + case mWorkerWatchdogMegaMiss: + worker(s, func(w *WorkerCounters) { w.WatchdogMegaMiss += s.Value }) + return + case mListenerWorkerCx: + // Summed across listeners; the admin listener reports elsewhere, so + // everything arriving here is real traffic. + worker(s, func(w *WorkerCounters) { w.AcceptedCx += s.Value }) + return + case mWorkerLoopUsSum: + worker(s, func(w *WorkerCounters) { w.LoopDurUsSum += s.Value }) + return + case mWorkerLoopUsCount: + worker(s, func(w *WorkerCounters) { w.LoopDurUsCount += s.Value }) + return + case mServerConcurrency: + out.Concurrency = s.Value + return + case mServerMemoryAllocated: + out.MemoryAllocated = s.Value + return + case mServerMemoryHeapSize: + out.MemoryHeapSize = s.Value + return + case mServerTotalConns: + out.TotalConnections = s.Value + return + case mDownstreamCxActive: + out.DownstreamCxActive += s.Value + return + case mDownstreamRqTotal: + out.DownstreamRqTotal += s.Value + return + case mDownstreamRqTimeSum: + if s.Labels["envoy_http_conn_manager_prefix"] != adminConnManagerPrefix { + out.DownstreamRqTimeMsTotal += s.Value + } + return + case mDownstreamRqTimeCount: + if s.Labels["envoy_http_conn_manager_prefix"] != adminConnManagerPrefix { + out.DownstreamRqTimeCount += s.Value + } + return + } + + name := s.Labels["envoy_cluster_name"] + if name == "" { + return + } + c := out.Clusters[name] + switch s.Name { + case mClusterCxActive: + c.CxActive = s.Value + case mClusterCxTotal: + c.CxTotal = s.Value + case mClusterCxOverflow: + c.CxOverflow = s.Value + case mClusterCxConnectFail: + c.CxConnectFail = s.Value + case mClusterCxConnectTimeout: + c.CxConnectTimeout = s.Value + case mClusterRqActive: + c.RqActive = s.Value + case mClusterRqTotal: + c.RqTotal = s.Value + case mClusterRqTimeout: + c.RqTimeout = s.Value + case mClusterRqRetry: + c.RqRetry = s.Value + case mClusterRqPendingActive: + c.RqPendingActive = s.Value + case mClusterRqPendingOverflw: + c.RqPendingOverflow = s.Value + case mClusterCbCxOpen: + c.CbCxOpen = s.Value + case mClusterCbRqOpen: + c.CbRqOpen = s.Value + case mClusterCbPendingOpen: + c.CbPendingOpen = s.Value + case mClusterRqTimeSum: + c.RqTimeMsTotal = s.Value + case mClusterRqTimeCount: + c.RqTimeCount = s.Value + case mClusterH2StreamsActive: + c.H2StreamsActive = s.Value + case mClusterH2PendingSend: + c.H2PendingSendBytes = s.Value + } + out.Clusters[name] = c + }) + if err != nil { + return EnvoyStats{}, err + } + return out, nil +} + +// ClusterDelta is one cluster's behavior over a window: gauges as read at the +// close, counters as differences. +type ClusterDelta struct { + Cluster string `json:"cluster"` + + CxActive float64 `json:"cx_active"` + RqActive float64 `json:"rq_active"` + RqPendingActive float64 `json:"rq_pending_active"` + + // CircuitBreakerOpen is true when Envoy reported any default-priority + // breaker open at the close of the window. Distinguishes "the router is + // slow" from "the router is refusing work it was configured not to do". + CircuitBreakerOpen bool `json:"circuit_breaker_open"` + + NewConnections float64 `json:"new_connections"` + Requests float64 `json:"requests"` + CxOverflow float64 `json:"cx_overflow"` + CxConnectFail float64 `json:"cx_connect_fail"` + CxConnectTimeout float64 `json:"cx_connect_timeout"` + RqTimeout float64 `json:"rq_timeout"` + RqRetry float64 `json:"rq_retry"` + RqPendingOverflow float64 `json:"rq_pending_overflow"` + + // RqPerCx is requests per upstream connection, cumulative since Envoy + // started; near 1 means every request opens its own connection and the port + // budget binds at the request rate. Cumulative because the per-window ratio + // is undefined in precisely the healthy case (no new connections). + RqPerCx float64 `json:"rq_per_cx"` + + // WindowRqPerCx is the same ratio confined to this window, nil when the + // window opened no connections. + WindowRqPerCx *float64 `json:"window_rq_per_cx,omitempty"` + + NewConnectionsPerSec float64 `json:"new_connections_per_sec"` + + // MeanRqTimeMs is the average request's time on this cluster's hop this + // window, over RqTimeSamples requests. Zero samples means no rq_time at + // all, the ext_proc cluster's permanent state (see ClusterStats.RqTimeMsTotal). + MeanRqTimeMs float64 `json:"mean_rq_time_ms"` + RqTimeSamples float64 `json:"rq_time_samples"` + + // H2StreamsActive and H2PendingSendBytes are gauges at the closing scrape, + // only populated for http2 clusters. On ate-cluster, streams piling up + // means requests are with the sidecar; pending bytes means they are stuck + // behind connection-level flow control before it. + H2StreamsActive float64 `json:"http2_streams_active,omitempty"` + H2PendingSendBytes float64 `json:"http2_pending_send_bytes,omitempty"` +} + +// WorkerDelta is one Envoy worker thread's behavior over a window. The point +// of the per-worker view is skew: sums and means over threads are already +// elsewhere, and they are exactly what hides one pinned worker among idle ones. +type WorkerDelta struct { + ID string `json:"id"` + // AcceptedCx is how many downstream connections this worker accepted this + // window. A connection stays on its worker for life, so persistent skew + // here becomes persistent load skew. + AcceptedCx float64 `json:"accepted_cx"` + // WatchdogMiss / WatchdogMegaMiss count event-loop stalls past 200ms / 1s. + WatchdogMiss float64 `json:"watchdog_miss"` + WatchdogMegaMiss float64 `json:"watchdog_mega_miss"` + // MeanLoopUs is the mean event-loop iteration time in microseconds over + // LoopSamples iterations. Zero samples means dispatcher stats are not + // enabled in the bootstrap, not that the loop never ran. + MeanLoopUs float64 `json:"mean_loop_us"` + LoopSamples float64 `json:"loop_samples"` +} + +// EnvoyDelta is the whole proxy's behavior over a window. +type EnvoyDelta struct { + Concurrency float64 `json:"concurrency"` + MemoryAllocated float64 `json:"memory_allocated"` + MemoryHeapSize float64 `json:"memory_heap_size"` + DownstreamCxActive float64 `json:"downstream_cx_active"` + DownstreamRq float64 `json:"downstream_rq"` + Clusters map[string]ClusterDelta `json:"clusters"` + + // MeanInEnvoyMs is the mean time a request spent inside Envoy during the + // window, admin traffic excluded, and InEnvoySamples is the request count it + // is over. This is the span every other hop is subtracted from. + MeanInEnvoyMs float64 `json:"mean_in_envoy_ms"` + InEnvoySamples float64 `json:"in_envoy_samples"` + + // Workers is ordered by worker id. Empty on an Envoy that predates the + // per-worker listener stats rather than zero-filled. + Workers []WorkerDelta `json:"workers,omitempty"` + + // Contention comes from the admin /contention endpoint, a separate fetch + // from the Prometheus scrape, and is attached here after the delta is + // built. Nil when the fetch failed; Enabled=false when the proxy runs + // without --enable-mutex-tracing. + Contention *ContentionDelta `json:"contention,omitempty"` +} + +// envoyDelta differences two scrapes. A counter that went backwards means +// Envoy restarted between scrapes, which invalidates the window rather than +// producing a negative rate. +func envoyDelta(prev, cur EnvoyStats, secs float64) (EnvoyDelta, error) { + if secs <= 0 { + return EnvoyDelta{}, fmt.Errorf("envoy delta over a non-positive interval") + } + d := EnvoyDelta{ + Concurrency: cur.Concurrency, + MemoryAllocated: cur.MemoryAllocated, + MemoryHeapSize: cur.MemoryHeapSize, + DownstreamCxActive: cur.DownstreamCxActive, + Clusters: map[string]ClusterDelta{}, + } + if cur.DownstreamRqTotal < prev.DownstreamRqTotal { + return EnvoyDelta{}, fmt.Errorf("envoy downstream_rq_total went backwards (%.0f to %.0f): the proxy restarted mid-window", + prev.DownstreamRqTotal, cur.DownstreamRqTotal) + } + d.DownstreamRq = cur.DownstreamRqTotal - prev.DownstreamRqTotal + + if n := cur.DownstreamRqTimeCount - prev.DownstreamRqTimeCount; n > 0 { + d.InEnvoySamples = n + d.MeanInEnvoyMs = (cur.DownstreamRqTimeMsTotal - prev.DownstreamRqTimeMsTotal) / n + } + + for name, c := range cur.Clusters { + p := prev.Clusters[name] + if c.RqTotal < p.RqTotal || c.CxTotal < p.CxTotal { + return EnvoyDelta{}, fmt.Errorf("envoy cluster %q counters went backwards: the proxy restarted mid-window", name) + } + cd := ClusterDelta{ + Cluster: name, + CxActive: c.CxActive, + RqActive: c.RqActive, + RqPendingActive: c.RqPendingActive, + CircuitBreakerOpen: c.CbCxOpen > 0 || c.CbRqOpen > 0 || c.CbPendingOpen > 0, + NewConnections: c.CxTotal - p.CxTotal, + Requests: c.RqTotal - p.RqTotal, + CxOverflow: c.CxOverflow - p.CxOverflow, + CxConnectFail: c.CxConnectFail - p.CxConnectFail, + CxConnectTimeout: c.CxConnectTimeout - p.CxConnectTimeout, + RqTimeout: c.RqTimeout - p.RqTimeout, + RqRetry: c.RqRetry - p.RqRetry, + RqPendingOverflow: c.RqPendingOverflow - p.RqPendingOverflow, + } + if c.CxTotal > 0 { + cd.RqPerCx = c.RqTotal / c.CxTotal + } + if cd.NewConnections > 0 { + w := cd.Requests / cd.NewConnections + cd.WindowRqPerCx = &w + } + if n := c.RqTimeCount - p.RqTimeCount; n > 0 { + cd.RqTimeSamples = n + cd.MeanRqTimeMs = (c.RqTimeMsTotal - p.RqTimeMsTotal) / n + } + cd.H2StreamsActive = c.H2StreamsActive + cd.H2PendingSendBytes = c.H2PendingSendBytes + cd.NewConnectionsPerSec = cd.NewConnections / secs + d.Clusters[name] = cd + } + + for id, w := range cur.Workers { + p := prev.Workers[id] + wd := WorkerDelta{ + ID: id, + AcceptedCx: w.AcceptedCx - p.AcceptedCx, + WatchdogMiss: w.WatchdogMiss - p.WatchdogMiss, + WatchdogMegaMiss: w.WatchdogMegaMiss - p.WatchdogMegaMiss, + } + if n := w.LoopDurUsCount - p.LoopDurUsCount; n > 0 { + wd.LoopSamples = n + wd.MeanLoopUs = (w.LoopDurUsSum - p.LoopDurUsSum) / n + } + d.Workers = append(d.Workers, wd) + } + // Numeric order, not lexicographic: "10" after "9", so the slice lines up + // with worker indices on an arm wider than ten. + sort.Slice(d.Workers, func(i, j int) bool { + a, _ := strconv.Atoi(d.Workers[i].ID) + b, _ := strconv.Atoi(d.Workers[j].ID) + return a < b + }) + return d, nil +} + +// ContentionStats is one read of Envoy's admin /contention endpoint, which +// only carries data when the proxy runs with --enable-mutex-tracing. It is one +// aggregate for the whole process, not per lock. +type ContentionStats struct { + At time.Time + Enabled bool + NumContentions float64 + LifetimeWaitCycles float64 +} + +// ContentionDelta is the window's share of the two cumulative counters. +type ContentionDelta struct { + // Enabled is false when the proxy runs without --enable-mutex-tracing, in + // which case the two counts are structurally zero rather than measured + // zeros. + Enabled bool `json:"enabled"` + // NumContentions is how many mutex acquisitions blocked this window. + NumContentions float64 `json:"num_contentions"` + // WaitCycles is the CPU cycles threads spent blocked on mutexes this + // window, summed across all locks and threads. Cycles, not seconds — read + // it relative to its own baseline, not as absolute time. + WaitCycles float64 `json:"wait_cycles"` +} + +func contentionDelta(prev, cur ContentionStats) ContentionDelta { + return ContentionDelta{ + Enabled: cur.Enabled, + NumContentions: cur.NumContentions - prev.NumContentions, + WaitCycles: cur.LifetimeWaitCycles - prev.LifetimeWaitCycles, + } +} + +// parseContention decodes the admin /contention JSON, accepting counters as +// numbers or strings (protobuf JSON renders uint64 as strings). The live v1.30 +// endpoint carries no "enabled" field, so Enabled is the counters' presence. +func parseContention(r io.Reader, at time.Time) (ContentionStats, error) { + var raw map[string]any + if err := json.NewDecoder(r).Decode(&raw); err != nil { + return ContentionStats{}, fmt.Errorf("decode /contention: %w", err) + } + num := func(v any) float64 { + switch x := v.(type) { + case float64: + return x + case string: + f, _ := strconv.ParseFloat(x, 64) + return f + } + return 0 + } + _, hasNum := raw["num_contentions"] + enabled := hasNum + if e, ok := raw["enabled"].(bool); ok { + enabled = e + } + return ContentionStats{ + At: at, + Enabled: enabled, + NumContentions: num(raw["num_contentions"]), + LifetimeWaitCycles: num(raw["lifetime_wait_cycles"]), + }, nil +} + +// ContentionClient reads Envoy's admin /contention endpoint. +type ContentionClient struct { + Fetch func(ctx context.Context) (io.ReadCloser, error) +} + +// Scrape reads one sample. +func (c *ContentionClient) Scrape(ctx context.Context) (ContentionStats, error) { + rc, err := c.Fetch(ctx) + if err != nil { + return ContentionStats{}, fmt.Errorf("fetch envoy contention: %w", err) + } + defer rc.Close() + return parseContention(rc, time.Now()) +} + +// EnvoyClient scrapes Envoy's admin Prometheus endpoint. +type EnvoyClient struct { + Fetch func(ctx context.Context) (io.ReadCloser, error) +} + +// Scrape reads one sample set. +func (c *EnvoyClient) Scrape(ctx context.Context) (EnvoyStats, error) { + rc, err := c.Fetch(ctx) + if err != nil { + return EnvoyStats{}, fmt.Errorf("fetch envoy stats: %w", err) + } + defer rc.Close() + return parseEnvoyStats(rc, time.Now()) +} + +// RouterStats is the subset of the Go sidecar's own metrics the run cares +// about: parking, and how long the sidecar holds each request. The +// route-duration histogram is the only measurement of the Envoy-to-sidecar hop +// that exists anywhere — Envoy publishes no timer for its ext_proc callout. +type RouterStats struct { + At time.Time + // ParkingActive is the live count of parked requests. + ParkingActive float64 + // ParkingRejectedTotal counts requests shed because the lot was full, + // cumulative since process start. + ParkingRejectedTotal float64 + // ParkingWaitSecondsTotal and ParkingWaitCount are the histogram's sum and + // count. The count is not "requests that had to wait": the router takes a + // slot around every ResumeActor call (extproc.go handleRequestHeaders), so + // the observation is the resume round-trip, waiting or not. + ParkingWaitSecondsTotal float64 + ParkingWaitCount float64 + + // RouteSecondsTotal and RouteCount are the atenet.router.route.duration + // histogram's sum and count — the whole time the ext_proc handler holds a + // request, summed across every outcome label, not just "ok". + // ParkingWaitSecondsTotal is nested inside this; the two must never be + // added together. + RouteSecondsTotal float64 + RouteCount float64 + + // Found records whether any parking series was present at all, so a + // renamed metric shows up as "not measured" instead of "always zero". + Found bool + // RouteFound is the same guarantee for the route-duration series, tracked + // separately because the two instruments can be renamed independently. + RouteFound bool +} + +// Metric-name prefixes, matched by prefix rather than spelled exactly because +// the OpenTelemetry Prometheus exporter appends unit and _total suffixes and +// rewrites dots. +const ( + parkingPrefix = "atenet_router_parking" + routePrefix = "atenet_router_route" +) + +// parseRouterStats extracts the parking and route-duration series from the +// sidecar's own Prometheus endpoint, classifying series by substring rather +// than exact name for the reason given above. +func parseRouterStats(r io.Reader, at time.Time) (RouterStats, error) { + out := RouterStats{At: at} + err := scanPromTextMatch(r, + func(name string) bool { + return strings.HasPrefix(name, parkingPrefix) || strings.HasPrefix(name, routePrefix) + }, + func(s promSample) { + name := s.Name + if strings.HasSuffix(name, "_bucket") { + return + } + if strings.HasPrefix(name, routePrefix) { + out.RouteFound = true + switch { + case strings.HasSuffix(name, "_sum"): + out.RouteSecondsTotal += s.Value + case strings.HasSuffix(name, "_count"): + out.RouteCount += s.Value + } + return + } + out.Found = true + switch { + case strings.Contains(name, "rejected"): + out.ParkingRejectedTotal += s.Value + case strings.Contains(name, "wait") && strings.HasSuffix(name, "_sum"): + out.ParkingWaitSecondsTotal += s.Value + case strings.Contains(name, "wait") && strings.HasSuffix(name, "_count"): + out.ParkingWaitCount += s.Value + case strings.Contains(name, "active"): + out.ParkingActive += s.Value + } + }) + if err != nil { + return RouterStats{}, err + } + return out, nil +} + +// RouterClient scrapes the atenet-router sidecar's metrics endpoint. +type RouterClient struct { + Fetch func(ctx context.Context) (io.ReadCloser, error) +} + +// Scrape reads one sample set. +func (c *RouterClient) Scrape(ctx context.Context) (RouterStats, error) { + rc, err := c.Fetch(ctx) + if err != nil { + return RouterStats{}, fmt.Errorf("fetch router stats: %w", err) + } + defer rc.Close() + return parseRouterStats(rc, time.Now()) +} + +// RouterDelta is the parking behavior over one window. Read ParkingRejected +// first: it is the only field that says parking went wrong. +type RouterDelta struct { + Measured bool `json:"measured"` + // ParkingActive is the instantaneous slot occupancy at the closing scrape. + // By Little's Law it is roughly MeanResumeMs x request rate, so single + // digits at a few thousand QPS is healthy, not a backlog. + ParkingActive float64 `json:"parking_active"` + // ParkingRejected counts requests shed this window because the lot was + // full. It must stay zero: non-zero means the router answered 503 rather + // than routing. + ParkingRejected float64 `json:"parking_rejected"` + // ResumeCalls is how many requests completed a resume in this window, + // which equals the window's request count in a healthy run. Named for what + // it is because the underlying "parking" metric name inverts its meaning. + ResumeCalls float64 `json:"resume_calls"` + // MeanResumeMs is the mean time a request spent holding a parking slot, + // i.e. the ResumeActor round trip to ate-api-server. It is part of every + // request's client-observed latency — a component of p50, not a parking + // problem. + MeanResumeMs float64 `json:"mean_resume_ms"` + + // MeanRouteMs is the whole time the sidecar held the average request this + // window, and RouteCalls the number of requests it is over. The resume + // nests inside the route per request, but the two means are over different + // populations, so subtracting them is only valid while the counts agree. + MeanRouteMs float64 `json:"mean_route_ms"` + RouteCalls float64 `json:"route_calls"` + // RouteMeasured is false when the sidecar exposed no route-duration series, + // which collapses the span breakdown back to "sidecar and Envoy, fused". + RouteMeasured bool `json:"route_measured"` +} + +func routerDelta(prev, cur RouterStats) RouterDelta { + d := RouterDelta{ + Measured: cur.Found, + RouteMeasured: cur.RouteFound, + ParkingActive: cur.ParkingActive, + ParkingRejected: cur.ParkingRejectedTotal - prev.ParkingRejectedTotal, + ResumeCalls: cur.ParkingWaitCount - prev.ParkingWaitCount, + } + if d.ResumeCalls > 0 { + d.MeanResumeMs = (cur.ParkingWaitSecondsTotal - prev.ParkingWaitSecondsTotal) / d.ResumeCalls * 1000 + } + if n := cur.RouteCount - prev.RouteCount; n > 0 { + d.RouteCalls = n + d.MeanRouteMs = (cur.RouteSecondsTotal - prev.RouteSecondsTotal) / n * 1000 + } + return d +} diff --git a/internal/benchmarking/routercap/envoy_test.go b/internal/benchmarking/routercap/envoy_test.go new file mode 100644 index 000000000..489d85351 --- /dev/null +++ b/internal/benchmarking/routercap/envoy_test.go @@ -0,0 +1,548 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for Envoy and sidecar stats parsing and for the counter deltas. + +package routercap + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// envoyFixture renders an admin /stats/prometheus payload in Envoy's own +// shape: HELP/TYPE headers, no per-sample timestamps, the cluster name carried +// in a label, and an unrelated cluster present so label routing is exercised. +func envoyFixture(cxTotal, rqTotal, cxActive float64) string { + var b strings.Builder + fmt.Fprintf(&b, `# TYPE envoy_server_concurrency gauge +envoy_server_concurrency{} 40 +# TYPE envoy_server_memory_allocated gauge +envoy_server_memory_allocated{} 104857600 +envoy_server_memory_heap_size{} 209715200 +envoy_server_total_connections{} 512 +# TYPE envoy_http_downstream_cx_active gauge +envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 128 +envoy_http_downstream_rq_total{envoy_http_conn_manager_prefix="ingress_http"} 900000 +# TYPE envoy_cluster_upstream_cx_total counter +envoy_cluster_upstream_cx_total{envoy_cluster_name="actor_original_dst"} %.0f +envoy_cluster_upstream_rq_total{envoy_cluster_name="actor_original_dst"} %.0f +envoy_cluster_upstream_cx_active{envoy_cluster_name="actor_original_dst"} %.0f +envoy_cluster_upstream_rq_active{envoy_cluster_name="actor_original_dst"} 310 +envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="actor_original_dst"} 4 +envoy_cluster_upstream_cx_overflow{envoy_cluster_name="actor_original_dst"} 7 +envoy_cluster_upstream_cx_connect_fail{envoy_cluster_name="actor_original_dst"} 2 +envoy_cluster_upstream_cx_connect_timeout{envoy_cluster_name="actor_original_dst"} 1 +envoy_cluster_upstream_rq_timeout{envoy_cluster_name="actor_original_dst"} 3 +envoy_cluster_upstream_rq_retry{envoy_cluster_name="actor_original_dst"} 5 +envoy_cluster_upstream_rq_pending_overflow{envoy_cluster_name="actor_original_dst"} 9 +envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="actor_original_dst"} 0 +envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="actor_original_dst"} 0 +envoy_cluster_circuit_breakers_default_rq_pending_open{envoy_cluster_name="actor_original_dst"} 0 +envoy_cluster_upstream_cx_total{envoy_cluster_name="ate-cluster"} 16 +envoy_cluster_upstream_rq_total{envoy_cluster_name="ate-cluster"} %.0f +envoy_cluster_upstream_cx_active{envoy_cluster_name="ate-cluster"} 16 +envoy_cluster_upstream_rq_active{envoy_cluster_name="ate-cluster"} 300 +envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="ate-cluster"} 0 +# a metric we do not ask for, which must not appear anywhere +envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name="actor_original_dst"} 123456789 +`, cxTotal, rqTotal, cxActive, rqTotal) + + // Request-time histograms in Envoy's live shape: the admin listener + // reports its own series, the ext_proc cluster emits no rq_time, and the + // buckets must be ignored. The sums are rigged so the actor hop means + // exactly 4ms and the in-Envoy total exactly 20ms, at any rqTotal. + fmt.Fprintf(&b, `# TYPE envoy_http_downstream_rq_time histogram +envoy_http_downstream_rq_time_bucket{envoy_http_conn_manager_prefix="ingress_http",le="5"} 1234 +envoy_http_downstream_rq_time_sum{envoy_http_conn_manager_prefix="ingress_http"} %.0f +envoy_http_downstream_rq_time_count{envoy_http_conn_manager_prefix="ingress_http"} %.0f +envoy_http_downstream_rq_time_sum{envoy_http_conn_manager_prefix="admin"} 100 +envoy_http_downstream_rq_time_count{envoy_http_conn_manager_prefix="admin"} 100000 +# TYPE envoy_cluster_upstream_rq_time histogram +envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="actor_original_dst",le="5"} 4321 +envoy_cluster_upstream_rq_time_sum{envoy_cluster_name="actor_original_dst"} %.0f +envoy_cluster_upstream_rq_time_count{envoy_cluster_name="actor_original_dst"} %.0f +`, rqTotal*20, rqTotal, rqTotal*4, rqTotal) + + // The per-worker and http2 series in Envoy's live shape: worker id in a + // label, accepts split per listener (8443 idle), the dispatcher loop + // histogram, and http2 gauges per h2 cluster. Worker 1's accepts and loop + // time scale off rqTotal so the deltas below are non-trivial. + fmt.Fprintf(&b, `# TYPE envoy_cluster_http2_streams_active gauge +envoy_cluster_http2_streams_active{envoy_cluster_name="ate-cluster"} 24 +envoy_cluster_http2_pending_send_bytes{envoy_cluster_name="ate-cluster"} 4096 +envoy_cluster_http2_streams_active{envoy_cluster_name="xds_cluster"} 1 +# TYPE envoy_server_worker_watchdog_miss counter +envoy_server_worker_watchdog_miss{envoy_worker_id="0"} 2 +envoy_server_worker_watchdog_miss{envoy_worker_id="1"} 0 +envoy_server_worker_watchdog_mega_miss{envoy_worker_id="0"} 1 +envoy_server_worker_watchdog_mega_miss{envoy_worker_id="1"} 0 +# TYPE envoy_listener_worker_downstream_cx_total counter +envoy_listener_worker_downstream_cx_total{envoy_worker_id="0",envoy_listener_address="0.0.0.0_8080"} %.0f +envoy_listener_worker_downstream_cx_total{envoy_worker_id="0",envoy_listener_address="0.0.0.0_8443"} 0 +envoy_listener_worker_downstream_cx_total{envoy_worker_id="1",envoy_listener_address="0.0.0.0_8080"} %.0f +envoy_listener_worker_downstream_cx_total{envoy_worker_id="1",envoy_listener_address="0.0.0.0_8443"} 0 +# TYPE envoy_listener_manager_worker_dispatcher_loop_duration_us histogram +envoy_listener_manager_worker_dispatcher_loop_duration_us_bucket{envoy_worker_id="0",le="100"} 50 +envoy_listener_manager_worker_dispatcher_loop_duration_us_sum{envoy_worker_id="0"} %.0f +envoy_listener_manager_worker_dispatcher_loop_duration_us_count{envoy_worker_id="0"} %.0f +envoy_listener_manager_worker_dispatcher_loop_duration_us_sum{envoy_worker_id="1"} %.0f +envoy_listener_manager_worker_dispatcher_loop_duration_us_count{envoy_worker_id="1"} %.0f +`, cxTotal/2, cxTotal, rqTotal*50, rqTotal, rqTotal*200, rqTotal) + return b.String() +} + +func TestParseEnvoyStats(t *testing.T) { + at := time.Unix(1700000000, 0) + got, err := parseEnvoyStats(strings.NewReader(envoyFixture(300, 900000, 295)), at) + if err != nil { + t.Fatalf("parseEnvoyStats: %v", err) + } + + if got.Concurrency != 40 { + t.Errorf("Concurrency = %v, want 40", got.Concurrency) + } + if got.MemoryAllocated != 104857600 || got.MemoryHeapSize != 209715200 { + t.Errorf("memory = %v/%v, want 104857600/209715200", got.MemoryAllocated, got.MemoryHeapSize) + } + if got.DownstreamCxActive != 128 { + t.Errorf("DownstreamCxActive = %v, want 128", got.DownstreamCxActive) + } + + // Three: actor, ext_proc, and the xds cluster that arrives only via its + // http2 gauge. Server-level metrics must not create a cluster entry. + if len(got.Clusters) != 3 { + t.Fatalf("got %d clusters, want 3: %v", len(got.Clusters), got.Clusters) + } + actor, ok := got.Clusters[ActorClusterName] + if !ok { + t.Fatalf("no %q cluster in %v", ActorClusterName, got.Clusters) + } + if actor.CxTotal != 300 || actor.RqTotal != 900000 || actor.CxActive != 295 { + t.Errorf("actor cluster cx/rq/active = %v/%v/%v, want 300/900000/295", + actor.CxTotal, actor.RqTotal, actor.CxActive) + } + if actor.CxOverflow != 7 || actor.RqPendingOverflow != 9 || actor.RqTimeout != 3 { + t.Errorf("actor overflow/pending_overflow/timeout = %v/%v/%v, want 7/9/3", + actor.CxOverflow, actor.RqPendingOverflow, actor.RqTimeout) + } + if got.Clusters[ExtProcClusterName].CxActive != 16 { + t.Errorf("ext_proc cx_active = %v, want 16", got.Clusters[ExtProcClusterName].CxActive) + } +} + +// TestParseEnvoyStatsExcludesAdminRequestTime pins the exclusion that keeps +// the harness from measuring itself: the admin listener's request times are +// this harness scraping /stats. The fixture's admin listener is deliberately +// large and fast so a regression shows as a wrong number. +func TestParseEnvoyStatsExcludesAdminRequestTime(t *testing.T) { + got, err := parseEnvoyStats(strings.NewReader(envoyFixture(300, 900000, 295)), time.Unix(1700000000, 0)) + if err != nil { + t.Fatalf("parseEnvoyStats: %v", err) + } + if got.DownstreamRqTimeCount != 900000 { + t.Errorf("DownstreamRqTimeCount = %v, want 900000 (the admin listener's 100000 must not be counted)", + got.DownstreamRqTimeCount) + } + if mean := got.DownstreamRqTimeMsTotal / got.DownstreamRqTimeCount; mean != 20 { + t.Errorf("in-Envoy mean = %v ms, want 20", mean) + } + if n := got.Clusters[ActorClusterName].RqTimeCount; n != 900000 { + t.Errorf("actor RqTimeCount = %v, want 900000", n) + } + // The ext_proc cluster genuinely publishes no rq_time: its streams end in + // a reset. The span code must tell that apart from a zero-millisecond mean. + if n := got.Clusters[ExtProcClusterName].RqTimeCount; n != 0 { + t.Errorf("ext_proc RqTimeCount = %v, want 0", n) + } +} + +// TestEnvoyDeltaSteadyStatePooling: with no new connections in the window the +// per-window requests-per-connection is undefined and must be absent, not +// zero. The cumulative ratio must still show pooling is in force. +func TestEnvoyDeltaSteadyStatePooling(t *testing.T) { + t0 := time.Unix(1700000000, 0) + prev, err := parseEnvoyStats(strings.NewReader(envoyFixture(6, 900000, 6)), t0) + if err != nil { + t.Fatalf("parse prev: %v", err) + } + // Ten seconds on: 100000 more requests, not one new connection. + cur, err := parseEnvoyStats(strings.NewReader(envoyFixture(6, 1000000, 6)), t0.Add(10*time.Second)) + if err != nil { + t.Fatalf("parse cur: %v", err) + } + + d, err := envoyDelta(prev, cur, 10) + if err != nil { + t.Fatalf("envoyDelta: %v", err) + } + actor := d.Clusters[ActorClusterName] + if actor.NewConnections != 0 { + t.Fatalf("NewConnections = %v, want 0", actor.NewConnections) + } + if actor.WindowRqPerCx != nil { + t.Errorf("WindowRqPerCx = %v, want nil: no connections were opened, so the ratio is undefined", *actor.WindowRqPerCx) + } + if want := 1000000.0 / 6.0; actor.RqPerCx != want { + t.Errorf("RqPerCx = %v, want %v: pooling must still be visible when the window opens nothing", actor.RqPerCx, want) + } +} + +func TestEnvoyDelta(t *testing.T) { + t0 := time.Unix(1700000000, 0) + prev, err := parseEnvoyStats(strings.NewReader(envoyFixture(300, 900000, 295)), t0) + if err != nil { + t.Fatalf("parse prev: %v", err) + } + // Ten seconds later: 100 new connections carried 100000 more requests. + cur, err := parseEnvoyStats(strings.NewReader(envoyFixture(400, 1000000, 295)), t0.Add(10*time.Second)) + if err != nil { + t.Fatalf("parse cur: %v", err) + } + + d, err := envoyDelta(prev, cur, 10) + if err != nil { + t.Fatalf("envoyDelta: %v", err) + } + actor := d.Clusters[ActorClusterName] + if actor.NewConnections != 100 { + t.Errorf("NewConnections = %v, want 100", actor.NewConnections) + } + if actor.Requests != 100000 { + t.Errorf("Requests = %v, want 100000", actor.Requests) + } + if actor.NewConnectionsPerSec != 10 { + t.Errorf("NewConnectionsPerSec = %v, want 10", actor.NewConnectionsPerSec) + } + // The pooling check the run depends on: far above 1 means connections are + // reused and port use tracks concurrency, not request rate. Cumulative, so + // 1000000 requests over 400 connections ever opened. + if actor.RqPerCx != 2500 { + t.Errorf("RqPerCx = %v, want 2500", actor.RqPerCx) + } + // The window's own ratio is present here because the window did open + // connections: 100000 requests over 100 of them. + if actor.WindowRqPerCx == nil || *actor.WindowRqPerCx != 1000 { + t.Errorf("WindowRqPerCx = %v, want 1000", actor.WindowRqPerCx) + } + // Counters that did not move must delta to zero, not carry their absolute + // value through. + if actor.CxOverflow != 0 || actor.RqTimeout != 0 { + t.Errorf("unchanged counters delta'd to %v/%v, want 0/0", actor.CxOverflow, actor.RqTimeout) + } + // Gauges are levels, read at the close. + if actor.CxActive != 295 { + t.Errorf("CxActive = %v, want 295", actor.CxActive) + } + if actor.CircuitBreakerOpen { + t.Error("CircuitBreakerOpen = true, want false: no breaker gauge was set in the fixture") + } + if d.DownstreamRq != 0 { + t.Errorf("DownstreamRq = %v, want 0", d.DownstreamRq) + } + if d.Concurrency != 40 { + t.Errorf("Concurrency = %v, want 40", d.Concurrency) + } +} + +// TestEnvoyDeltaPerWorker pins the per-thread view: accepts and watchdog +// misses delta per worker, loop duration comes out as a mean, and http2 +// gauges land on the owning cluster. The fields exist to expose skew, so the +// fixture's two workers are deliberately unequal. +func TestEnvoyDeltaPerWorker(t *testing.T) { + t0 := time.Unix(1700000000, 0) + prev, err := parseEnvoyStats(strings.NewReader(envoyFixture(300, 900000, 295)), t0) + if err != nil { + t.Fatalf("parse prev: %v", err) + } + cur, err := parseEnvoyStats(strings.NewReader(envoyFixture(400, 1000000, 295)), t0.Add(10*time.Second)) + if err != nil { + t.Fatalf("parse cur: %v", err) + } + d, err := envoyDelta(prev, cur, 10) + if err != nil { + t.Fatalf("envoyDelta: %v", err) + } + + if len(d.Workers) != 2 { + t.Fatalf("got %d workers, want 2: %v", len(d.Workers), d.Workers) + } + if d.Workers[0].ID != "0" || d.Workers[1].ID != "1" { + t.Fatalf("worker order = %s,%s, want 0,1", d.Workers[0].ID, d.Workers[1].ID) + } + // Fixture: worker 0 accepts cxTotal/2 on 8080 (150 -> 200), worker 1 + // accepts cxTotal (300 -> 400). The 8443 listener is idle and must not + // disturb the sum. + if d.Workers[0].AcceptedCx != 50 || d.Workers[1].AcceptedCx != 100 { + t.Errorf("AcceptedCx = %v/%v, want 50/100", d.Workers[0].AcceptedCx, d.Workers[1].AcceptedCx) + } + // Watchdog counters did not move between the scrapes: zero, not the + // absolute 2/1 carried through. + if d.Workers[0].WatchdogMiss != 0 || d.Workers[0].WatchdogMegaMiss != 0 { + t.Errorf("watchdog deltas = %v/%v, want 0/0", + d.Workers[0].WatchdogMiss, d.Workers[0].WatchdogMegaMiss) + } + // Loop histogram: sum grows 50us per iteration for worker 0 and 200us for + // worker 1 — the skewed pair the field exists to expose. + if d.Workers[0].MeanLoopUs != 50 || d.Workers[1].MeanLoopUs != 200 { + t.Errorf("MeanLoopUs = %v/%v, want 50/200", d.Workers[0].MeanLoopUs, d.Workers[1].MeanLoopUs) + } + if d.Workers[0].LoopSamples != 100000 { + t.Errorf("LoopSamples = %v, want 100000", d.Workers[0].LoopSamples) + } + + ext := d.Clusters[ExtProcClusterName] + if ext.H2StreamsActive != 24 || ext.H2PendingSendBytes != 4096 { + t.Errorf("ate-cluster h2 streams/pending = %v/%v, want 24/4096", + ext.H2StreamsActive, ext.H2PendingSendBytes) + } + if d.Clusters[ActorClusterName].H2StreamsActive != 0 { + t.Error("actor cluster picked up an http2 gauge; it is an http1 cluster") + } +} + +// TestParseContention covers the two spellings the admin server uses for +// uint64 counters — protobuf JSON renders them as strings — and the delta +// carrying Enabled through so a zero from a proxy without mutex tracing is +// distinguishable from a measured zero. +func TestParseContention(t *testing.T) { + at := time.Unix(1700000000, 0) + cur, err := parseContention(strings.NewReader( + `{"enabled": true, "num_contentions": "150", "current_wait_cycles": "7", "lifetime_wait_cycles": "90000"}`), at) + if err != nil { + t.Fatalf("parseContention: %v", err) + } + if !cur.Enabled || cur.NumContentions != 150 || cur.LifetimeWaitCycles != 90000 { + t.Errorf("parsed %+v, want enabled/150/90000", cur) + } + // Bare numbers must parse too, and an explicit enabled:false wins over the + // counters' presence. + n, err := parseContention(strings.NewReader( + `{"enabled": false, "num_contentions": 3, "lifetime_wait_cycles": 12}`), at) + if err != nil { + t.Fatalf("parseContention (numeric): %v", err) + } + if n.Enabled || n.NumContentions != 3 { + t.Errorf("parsed %+v, want disabled/3", n) + } + // The live v1.30 shape: three counters, no "enabled" key. Their presence is + // the signal that tracing is on. + live, err := parseContention(strings.NewReader( + `{"num_contentions": "123", "current_wait_cycles": "13516", "lifetime_wait_cycles": "8413526"}`), at) + if err != nil { + t.Fatalf("parseContention (live shape): %v", err) + } + if !live.Enabled || live.LifetimeWaitCycles != 8413526 { + t.Errorf("parsed %+v, want enabled/8413526", live) + } + + d := contentionDelta(ContentionStats{NumContentions: 100, LifetimeWaitCycles: 40000}, cur) + if d.NumContentions != 50 || d.WaitCycles != 50000 || !d.Enabled { + t.Errorf("delta = %+v, want 50/50000/enabled", d) + } +} + +func TestEnvoyDeltaRejectsARestart(t *testing.T) { + t0 := time.Unix(1700000000, 0) + prev, _ := parseEnvoyStats(strings.NewReader(envoyFixture(300, 900000, 295)), t0) + // Envoy restarted: counters reset to near zero. Differencing would produce + // a large negative rate, which must not reach the output. + cur, _ := parseEnvoyStats(strings.NewReader(envoyFixture(4, 120, 4)), t0.Add(10*time.Second)) + + if _, err := envoyDelta(prev, cur, 10); err == nil { + t.Fatal("envoyDelta accepted counters that went backwards; want an error naming the restart") + } else if !strings.Contains(err.Error(), "restarted") { + t.Errorf("error = %q, want it to name the restart", err) + } +} + +func TestEnvoyDeltaFlagsAnOpenBreaker(t *testing.T) { + t0 := time.Unix(1700000000, 0) + prev, _ := parseEnvoyStats(strings.NewReader(envoyFixture(300, 900000, 295)), t0) + body := strings.Replace(envoyFixture(400, 1000000, 20000), + `envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="actor_original_dst"} 0`, + `envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="actor_original_dst"} 1`, 1) + cur, _ := parseEnvoyStats(strings.NewReader(body), t0.Add(10*time.Second)) + + d, err := envoyDelta(prev, cur, 10) + if err != nil { + t.Fatalf("envoyDelta: %v", err) + } + if !d.Clusters[ActorClusterName].CircuitBreakerOpen { + t.Error("CircuitBreakerOpen = false, want true: rq_open was 1") + } +} + +func TestEnvoyDeltaRejectsANonPositiveInterval(t *testing.T) { + if _, err := envoyDelta(EnvoyStats{}, EnvoyStats{}, 0); err == nil { + t.Fatal("envoyDelta accepted a zero-length interval") + } +} + +// routerFixture uses the names the OpenTelemetry Prometheus exporter produces +// today: dots become underscores, the histogram gains a unit suffix, and the +// counter gains _total. Route duration is split across outcome labels (summed: +// 5s over 2000 requests), and the parking 2.4s over 1000 resumes nests inside +// the 4s the handler spent on those requests so a sign error cannot pass. +const routerFixture = `# HELP atenet_router_parking_active Requests currently parked. +# TYPE atenet_router_parking_active gauge +atenet_router_parking_active{otel_scope_name="atenet-router"} 12 +# TYPE atenet_router_parking_rejected_total counter +atenet_router_parking_rejected_total{otel_scope_name="atenet-router"} 40 +# TYPE atenet_router_parking_wait_duration_seconds histogram +atenet_router_parking_wait_duration_seconds_bucket{le="0.1"} 970 +atenet_router_parking_wait_duration_seconds_bucket{le="+Inf"} 1000 +atenet_router_parking_wait_duration_seconds_sum{otel_scope_name="atenet-router"} 2.4 +atenet_router_parking_wait_duration_seconds_count{otel_scope_name="atenet-router"} 1000 +# TYPE atenet_router_route_duration_seconds histogram +atenet_router_route_duration_seconds_bucket{ate_router_outcome="ok",le="0.01"} 700 +atenet_router_route_duration_seconds_sum{ate_router_outcome="ok",otel_scope_name="atenet-router"} 4 +atenet_router_route_duration_seconds_count{ate_router_outcome="ok",otel_scope_name="atenet-router"} 1000 +atenet_router_route_duration_seconds_sum{ate_router_outcome="cancelled",otel_scope_name="atenet-router"} 1 +atenet_router_route_duration_seconds_count{ate_router_outcome="cancelled",otel_scope_name="atenet-router"} 100 +atenet_router_route_duration_seconds_sum{ate_router_outcome="no_capacity",otel_scope_name="atenet-router"} 0 +atenet_router_route_duration_seconds_count{ate_router_outcome="no_capacity",otel_scope_name="atenet-router"} 900 +# unrelated series that must not be picked up +go_goroutines 250 +` + +func TestParseRouterStats(t *testing.T) { + got, err := parseRouterStats(strings.NewReader(routerFixture), time.Unix(1700000000, 0)) + if err != nil { + t.Fatalf("parseRouterStats: %v", err) + } + if !got.Found { + t.Fatal("Found = false, want true") + } + if got.ParkingActive != 12 { + t.Errorf("ParkingActive = %v, want 12", got.ParkingActive) + } + if got.ParkingRejectedTotal != 40 { + t.Errorf("ParkingRejectedTotal = %v, want 40", got.ParkingRejectedTotal) + } + // Buckets are cumulative and would inflate the count if summed in. + if got.ParkingWaitCount != 1000 { + t.Errorf("ParkingWaitCount = %v, want 1000 (buckets must be skipped)", got.ParkingWaitCount) + } + if got.ParkingWaitSecondsTotal != 2.4 { + t.Errorf("ParkingWaitSecondsTotal = %v, want 2.4", got.ParkingWaitSecondsTotal) + } + // Route duration must sum across every outcome label. Reading only the "ok" + // series would drop the requests the sidecar cancelled or shed, which are + // exactly the requests whose time is most worth attributing. + if !got.RouteFound { + t.Fatal("RouteFound = false, want true") + } + if got.RouteCount != 2000 { + t.Errorf("RouteCount = %v, want 2000 (1000 ok + 100 cancelled + 900 no_capacity)", got.RouteCount) + } + if got.RouteSecondsTotal != 5 { + t.Errorf("RouteSecondsTotal = %v, want 5", got.RouteSecondsTotal) + } +} + +// TestParseRouterStatsSeparatesTheTwoInstruments: parking wait nests inside +// route duration, so adding them would double-count the resume. +func TestParseRouterStatsSeparatesTheTwoInstruments(t *testing.T) { + got, err := parseRouterStats(strings.NewReader(routerFixture), time.Unix(1700000000, 0)) + if err != nil { + t.Fatalf("parseRouterStats: %v", err) + } + if got.ParkingWaitCount == got.RouteCount { + t.Error("parking and route counts are equal: the two instruments were merged") + } + if got.ParkingWaitSecondsTotal >= got.RouteSecondsTotal { + t.Errorf("parking wait %vs is not inside route duration %vs", + got.ParkingWaitSecondsTotal, got.RouteSecondsTotal) + } +} + +// TestParseRouterStatsTracksTheTwoPrefixesIndependently covers a sidecar that +// exposes parking but not route duration — an older router image against a +// newer harness. The breakdown degrades to "sidecar and Envoy fused" rather +// than reporting a confident zero for the sidecar hop. +func TestParseRouterStatsTracksTheTwoPrefixesIndependently(t *testing.T) { + parkingOnly := `atenet_router_parking_active{otel_scope_name="atenet-router"} 3 +atenet_router_parking_wait_duration_seconds_count{otel_scope_name="atenet-router"} 50 +` + got, err := parseRouterStats(strings.NewReader(parkingOnly), time.Unix(1700000000, 0)) + if err != nil { + t.Fatalf("parseRouterStats: %v", err) + } + if !got.Found { + t.Error("Found = false, want true: parking series were present") + } + if got.RouteFound { + t.Error("RouteFound = true on a payload with no route series") + } + if d := routerDelta(RouterStats{}, got); d.RouteMeasured { + t.Error("RouterDelta.RouteMeasured = true, want false") + } +} + +func TestParseRouterStatsReportsARenamedMetricAsUnmeasured(t *testing.T) { + // If the exporter's naming ever moves out from under parkingPrefix, the + // output must say "not measured" rather than a confident zero. + got, err := parseRouterStats(strings.NewReader("go_goroutines 250\n"), time.Unix(1700000000, 0)) + if err != nil { + t.Fatalf("parseRouterStats: %v", err) + } + if got.Found { + t.Fatal("Found = true on a payload with no parking series") + } + if d := routerDelta(RouterStats{}, got); d.Measured { + t.Error("RouterDelta.Measured = true, want false") + } +} + +func TestRouterDelta(t *testing.T) { + prev := RouterStats{Found: true, ParkingRejectedTotal: 40, ParkingWaitSecondsTotal: 25, ParkingWaitCount: 100} + cur := RouterStats{ + Found: true, + ParkingActive: 7, + ParkingRejectedTotal: 46, + ParkingWaitSecondsTotal: 32, + ParkingWaitCount: 120, + } + + d := routerDelta(prev, cur) + if !d.Measured { + t.Error("Measured = false, want true") + } + if d.ParkingActive != 7 { + t.Errorf("ParkingActive = %v, want 7", d.ParkingActive) + } + if d.ParkingRejected != 6 { + t.Errorf("ParkingRejected = %v, want 6", d.ParkingRejected) + } + if d.ResumeCalls != 20 { + t.Errorf("ResumeCalls = %v, want 20", d.ResumeCalls) + } + // 7 seconds of slot occupancy spread over 20 resumes. + if want := 350.0; d.MeanResumeMs != want { + t.Errorf("MeanResumeMs = %v, want %v", d.MeanResumeMs, want) + } +} + +func TestRouterDeltaWithNoResumesLeavesTheMeanAtZero(t *testing.T) { + // An idle window: two scrapes with no resume between them, which is what + // the setup and drain phases look like. A mean over zero samples must not + // divide by zero and surface as NaN in the JSON. + d := routerDelta(RouterStats{Found: true}, RouterStats{Found: true}) + if d.MeanResumeMs != 0 { + t.Errorf("MeanResumeMs = %v, want 0", d.MeanResumeMs) + } +} diff --git a/internal/benchmarking/routercap/guards.go b/internal/benchmarking/routercap/guards.go new file mode 100644 index 000000000..a5046eb93 --- /dev/null +++ b/internal/benchmarking/routercap/guards.go @@ -0,0 +1,312 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The guards that decide whether a window measured the router or measured the rig, +// and which of those failures are bad enough to abort the run. + +package routercap + +import ( + "fmt" + "log/slog" + "strconv" + "strings" +) + +// Each guard watches one way the harness can become the bottleneck, so a +// capacity figure is never really a statement about the load generator. +// Envoy's port-exhaustion counters are deliberately absent: that cliff is what +// the run came to measure. + +// ClientStats is the generator's view of its own transport over one window. +type ClientStats struct { + // NewConnections is TCP connections the generator opened to the router. + // With keep-alive intact this stays near zero after the pool warms. + NewConnections float64 `json:"new_connections"` + // RequestsPerConnection is requests dispatched per new connection over the + // window. Near 1 means a socket per request, which hits the generator's own + // ephemeral-port ceiling long before the router hits anything. + RequestsPerConnection float64 `json:"requests_per_connection"` + // ConnectionsInUse is the pool's live connection count. + ConnectionsInUse int64 `json:"connections_in_use"` +} + +// GuardName identifies a rig guard. +type GuardName string + +// One guard per way the harness can become the bottleneck. A fatal trip marks +// the arm rig-limited from that rung on: the windows before it stand, the +// windows after it describe the harness. +const ( + // GuardLoadgenCPU: the generator container ran out of its own CPU, so the + // reported offered rate is no longer real. + GuardLoadgenCPU GuardName = "loadgen_cpu" + // GuardControlPlaneThrottle: ate-api-server (or anything else in the + // router's namespace) was CFS-throttled. Every request resumes through the + // control plane, so this is indistinguishable from a slow router. + GuardControlPlaneThrottle GuardName = "control_plane_throttle" + // GuardWorkerConnRate: new connections per worker pod exceeded what a + // worker's 60s-TIME_WAIT source-port pool can absorb (~470/s; the guard + // trips at 400), so the wall belongs to the workers, not the router. + GuardWorkerConnRate GuardName = "worker_conn_rate" + // GuardClientKeepAlive: requests per new connection fell below the floor, + // meaning keep-alive stopped holding and the generator is in a dial storm, + // racing its own port ceiling. + GuardClientKeepAlive GuardName = "client_keepalive" + // GuardClientPorts: the generator's live connection count neared its own + // source-port budget (80% of its measured ip_local_port_range — see + // ResolveClientCeiling). The next cliff would be the rig's, not the + // router's. + GuardClientPorts GuardName = "client_ports" + // GuardDispatchLag: the pacer fell behind its own schedule, so the offered + // rate on the x-axis was not the rate actually offered. Suspended while the + // system under test is demonstrably saturated. + GuardDispatchLag GuardName = "dispatch_lag" +) + +// defaultEphemeralPorts is the size of the Linux default source-port range +// (32768-60999). Only a fallback for the generator's ceiling; the router's +// range is always read from the live pod. +const defaultEphemeralPorts = 60999 - 32768 + 1 + +// GuardTrip is one guard firing on one window. +type GuardTrip struct { + Guard GuardName `json:"guard"` + Value float64 `json:"value"` + Threshold float64 `json:"threshold"` + // Fatal says whether the run should stop. A trip can be recorded without + // being fatal — see the saturation exception on dispatch lag. + Fatal bool `json:"fatal"` + Detail string `json:"detail"` +} + +// GuardConfig holds the thresholds, each settable by a flag on the binary. It +// is serialized into the run header, because a loosened threshold changes what +// the run's silence means. +type GuardConfig struct { + // LoadgenCPUUtilization is the generator container's use against its own + // limit. Past this the offered rate stops being real. + LoadgenCPUUtilization float64 `json:"loadgen_cpu_utilization"` + // WorkerNewConnsPerSec is the per-worker-pod new-connection rate ceiling. + // A worker pod's source-port pool recycles on a 60s TIME_WAIT, giving + // ~470/s; the default sits under it. + WorkerNewConnsPerSec float64 `json:"worker_new_conns_per_sec"` + // MinRequestsPerConnection is the generator-side keep-alive check. + MinRequestsPerConnection float64 `json:"min_requests_per_connection"` + // ClientConnectionCeiling is how many simultaneous connections the + // generator may hold to the router before its own source ports bind first; + // zero disables the check. The default is 80% of the generator's measured + // ip_local_port_range (ResolveClientCeiling), with the Linux-default + // assumption standing only until that read happens or when it fails. + ClientConnectionCeiling int `json:"client_connection_ceiling"` + // DispatchLagP95Ms is how far behind its own schedule the generator may + // fall before the offered rate is fiction. This and SaturationLatencyP95Ms + // key on p95 because the run only keeps p50 and p95 (see LatencyStats), + // while the thresholds kept their old p99-era values — deliberately harder + // to trip. + DispatchLagP95Ms float64 `json:"dispatch_lag_p95_ms"` + + // SaturationLatencyP95Ms and SaturationAchievedRatio define "the system is + // demonstrably saturated", which suspends the dispatch-lag guard. + SaturationLatencyP95Ms float64 `json:"saturation_latency_p95_ms"` + SaturationAchievedRatio float64 `json:"saturation_achieved_ratio"` + + // WorkerPods is how many worker pods the load is spread across, used to + // turn the cluster-wide connection rate into a per-pod one. + WorkerPods int `json:"worker_pods"` +} + +// DefaultGuardConfig returns the thresholds argued for in the design. +func DefaultGuardConfig() GuardConfig { + return GuardConfig{ + LoadgenCPUUtilization: 0.80, + WorkerNewConnsPerSec: 400, + MinRequestsPerConnection: 10, + ClientConnectionCeiling: defaultEphemeralPorts * 8 / 10, + DispatchLagP95Ms: 50, + SaturationLatencyP95Ms: 200, + SaturationAchievedRatio: 0.95, + } +} + +// ResolveClientCeiling replaces a default client-connection ceiling with 80% +// of this process's own measured source-port range, read once at startup from +// inside the generator pod so /proc answers for the right network namespace. +// A ceiling that is neither zero (disabled) nor the default is a flag override +// and is left alone; a failed read keeps the conservative default. +func (g *GuardConfig) ResolveClientCeiling(readFile func(string) ([]byte, error), log *slog.Logger) { + if g.ClientConnectionCeiling != defaultEphemeralPorts*8/10 { + return + } + b, err := readFile("/proc/sys/net/ipv4/ip_local_port_range") + if err != nil { + log.Warn("could not read own ip_local_port_range; the client-connection ceiling keeps the Linux-default assumption", "err", err) + return + } + f := strings.Fields(string(b)) + if len(f) != 2 { + log.Warn("unparseable ip_local_port_range; keeping the default ceiling", "content", string(b)) + return + } + lo, err1 := strconv.Atoi(f[0]) + hi, err2 := strconv.Atoi(f[1]) + if err1 != nil || err2 != nil || hi < lo { + log.Warn("unparseable ip_local_port_range; keeping the default ceiling", "content", string(b)) + return + } + g.ClientConnectionCeiling = (hi - lo + 1) * 8 / 10 + log.Info("client-connection ceiling from this pod's own port range", + "range", fmt.Sprintf("%d-%d", lo, hi), "ceiling", g.ClientConnectionCeiling) +} + +// saturated reports whether the system is visibly failing to keep up, in which +// case the generator being slow is a symptom rather than a cause. Without this +// exception the dispatch-lag guard would abort the ladder at exactly the +// saturated rungs the run exists to measure. +func (g GuardConfig) saturated(s *Sample) bool { + if g.SaturationLatencyP95Ms > 0 && s.Load.Latency.P95Ms >= g.SaturationLatencyP95Ms { + return true + } + if g.SaturationAchievedRatio > 0 && s.Load.OfferedQPS > 0 && + s.Load.AchievedQPS < g.SaturationAchievedRatio*s.Load.OfferedQPS { + return true + } + return false +} + +// Check evaluates every guard against one window and returns the trips. It +// never mutates the sample; the caller decides what a fatal trip means. +func (g GuardConfig) Check(s *Sample) []GuardTrip { + var trips []GuardTrip + + if lg, ok := s.Containers[RoleLoadgen]; ok && g.LoadgenCPUUtilization > 0 { + if lg.CPULimitCores <= 0 { + // An unlimited generator container cannot be checked against a + // utilization threshold, and skipping silently would leave the + // run's most important guard quietly disabled. + trips = append(trips, GuardTrip{ + Guard: GuardLoadgenCPU, Value: lg.CPUCores, Fatal: true, + Detail: "loadgen container has no CPU limit, so its headroom cannot be checked; give it requests == limits", + }) + } else if lg.CPUUtilization > g.LoadgenCPUUtilization { + trips = append(trips, GuardTrip{ + Guard: GuardLoadgenCPU, Value: lg.CPUUtilization, Threshold: g.LoadgenCPUUtilization, Fatal: true, + Detail: fmt.Sprintf("load generator at %.0f%% of its %.1f-core limit; the offered rate is no longer reliable", + lg.CPUUtilization*100, lg.CPULimitCores), + }) + } + } + + if cp, ok := s.Groups[RoleControlPlane]; ok && cp.ThrottledPeriods > 0 { + // Any throttling at all, because a throttled ate-api-server is + // indistinguishable from a slow router when seen from the client. + trips = append(trips, GuardTrip{ + Guard: GuardControlPlaneThrottle, Value: cp.ThrottledPeriods, Threshold: 0, Fatal: true, + Detail: fmt.Sprintf("%s throttled for %.0f periods (%.3fs), worst container %q; router latency would be partly the control plane's", + RoleControlPlane, cp.ThrottledPeriods, cp.ThrottledSeconds, cp.ThrottledMaxOf), + }) + } + + if s.Envoy != nil && g.WorkerPods > 0 && g.WorkerNewConnsPerSec > 0 { + if actor, ok := s.Envoy.Clusters[ActorClusterName]; ok { + // Cluster-wide mean divided by pod count, a fair stand-in for the + // per-pod maximum only because the run places exactly one actor per + // worker pod and dispatches uniformly. If either changes, so must + // this. + perPod := actor.NewConnectionsPerSec / float64(g.WorkerPods) + if perPod > g.WorkerNewConnsPerSec { + trips = append(trips, GuardTrip{ + Guard: GuardWorkerConnRate, Value: perPod, Threshold: g.WorkerNewConnsPerSec, Fatal: true, + Detail: fmt.Sprintf("mean %.0f new connections/sec per worker pod across %d pods; a pod's source ports recycle on a 60s TIME_WAIT", + perPod, g.WorkerPods), + }) + } + } + } + + // Skipped when the generator opened nothing (perfect reuse divides by + // zero) and when the window carried no scheduled load (the pre-warm window + // races the ladder start). The ratio is only meaningful over traffic the + // ladder asked for. + if g.MinRequestsPerConnection > 0 && s.Load.OfferedQPS > 0 && s.Client.NewConnections > 0 && + s.Client.RequestsPerConnection < g.MinRequestsPerConnection { + trips = append(trips, GuardTrip{ + Guard: GuardClientKeepAlive, Value: s.Client.RequestsPerConnection, Threshold: g.MinRequestsPerConnection, Fatal: true, + Detail: fmt.Sprintf("generator averaged %.1f requests per connection over %.0f new connections; keep-alive to the router is not holding, so the client will hit its own port ceiling first", + s.Client.RequestsPerConnection, s.Client.NewConnections), + }) + } + + if g.ClientConnectionCeiling > 0 && s.Client.ConnectionsInUse > int64(g.ClientConnectionCeiling) { + trips = append(trips, GuardTrip{ + Guard: GuardClientPorts, Value: float64(s.Client.ConnectionsInUse), Threshold: float64(g.ClientConnectionCeiling), Fatal: true, + Detail: fmt.Sprintf("generator holds %d connections to the router, past its own source-port headroom; the next cliff would be the load generator's, not the router's", + s.Client.ConnectionsInUse), + }) + } + + if g.DispatchLagP95Ms > 0 && s.Load.DispatchLag.P95Ms > g.DispatchLagP95Ms { + sat := g.saturated(s) + detail := fmt.Sprintf("generator was %.1fms behind its own schedule at p95; offered load for this window is not what the x-axis says", + s.Load.DispatchLag.P95Ms) + if sat { + detail = fmt.Sprintf("generator was %.1fms behind its own schedule at p95, but the system is saturated (p95 latency %.1fms, achieved %.0f of %.0f offered QPS): recorded, not fatal", + s.Load.DispatchLag.P95Ms, s.Load.Latency.P95Ms, s.Load.AchievedQPS, s.Load.OfferedQPS) + } + trips = append(trips, GuardTrip{ + Guard: GuardDispatchLag, Value: s.Load.DispatchLag.P95Ms, Threshold: g.DispatchLagP95Ms, + Fatal: !sat, Detail: detail, + }) + } + + return trips +} + +// AnyFatal reports whether any trip should stop the run. +func AnyFatal(trips []GuardTrip) bool { return len(FatalTrips(trips)) > 0 } + +// FatalTrips returns only the trips that should stop the run. +func FatalTrips(trips []GuardTrip) []GuardTrip { + var out []GuardTrip + for _, t := range trips { + if t.Fatal { + out = append(out, t) + } + } + return out +} + +// RigLimitedError is returned when a guard stops the run. It maps to exit code +// 3, distinct from a system failure, so automation can tell "we could not +// measure this" from "the router fell over". +type RigLimitedError struct { + Trips []GuardTrip +} + +func (e *RigLimitedError) Error() string { + if len(e.Trips) == 0 { + return "rig-limited" + } + msg := "rig-limited: " + for i, t := range e.Trips { + if !t.Fatal { + continue + } + if i > 0 { + msg += "; " + } + msg += string(t.Guard) + ": " + t.Detail + } + return msg +} diff --git a/internal/benchmarking/routercap/guards_test.go b/internal/benchmarking/routercap/guards_test.go new file mode 100644 index 000000000..beb7a9efb --- /dev/null +++ b/internal/benchmarking/routercap/guards_test.go @@ -0,0 +1,277 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for each guard's trip condition and for which trips are fatal. + +package routercap + +import ( + "fmt" + "io" + "log/slog" + "testing" +) + +// healthySample is a window where nothing is wrong: the generator is idle +// relative to its limit, keeping its schedule and reusing connections, and the +// control plane is not throttling. Each test perturbs exactly one thing. +func healthySample(cfg *GuardConfig) *Sample { + cfg.WorkerPods = 100 + return &Sample{ + Load: GenStats{ + OfferedQPS: 8000, + AchievedQPS: 8000, + Latency: LatencyStats{P95Ms: 12}, + DispatchLag: LatencyStats{P95Ms: 2}, + }, + Client: ClientStats{NewConnections: 4, RequestsPerConnection: 2000}, + Containers: map[string]ContainerUsage{ + RoleLoadgen: {Container: "loadgen", CPUCores: 20, CPULimitCores: 100, CPUUtilization: 0.20}, + RoleEnvoy: {Container: "envoy", CPUCores: 30, CPULimitCores: 40, CPUUtilization: 0.75}, + }, + Groups: map[string]GroupUsage{ + RoleControlPlane: {Containers: 8, CPUCores: 6}, + }, + Envoy: &EnvoyDelta{Clusters: map[string]ClusterDelta{ + ActorClusterName: {NewConnectionsPerSec: 100}, + }}, + } +} + +func tripFor(trips []GuardTrip, name GuardName) (GuardTrip, bool) { + for _, t := range trips { + if t.Guard == name { + return t, true + } + } + return GuardTrip{}, false +} + +func TestGuardsPassOnAHealthyWindow(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + if trips := cfg.Check(s); len(trips) != 0 { + t.Fatalf("healthy window tripped %d guards: %+v", len(trips), trips) + } +} + +func TestGuardLoadgenCPU(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Containers[RoleLoadgen] = ContainerUsage{Container: "loadgen", CPUCores: 85, CPULimitCores: 100, CPUUtilization: 0.85} + + trips := cfg.Check(s) + tr, ok := tripFor(trips, GuardLoadgenCPU) + if !ok { + t.Fatalf("loadgen at 85%% of its limit did not trip: %+v", trips) + } + if !tr.Fatal { + t.Error("loadgen CPU trip is not fatal; the offered rate is unreliable past this point") + } +} + +func TestGuardLoadgenCPUWithoutALimitIsItselfATrip(t *testing.T) { + // An unlimited generator container silently disables the run's most + // important guard, so it has to be an error rather than a skip. + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Containers[RoleLoadgen] = ContainerUsage{Container: "loadgen", CPUCores: 3} + + tr, ok := tripFor(cfg.Check(s), GuardLoadgenCPU) + if !ok || !tr.Fatal { + t.Fatalf("an unlimited loadgen container must trip fatally; got ok=%v trip=%+v", ok, tr) + } +} + +func TestGuardControlPlaneThrottleTripsOnAnyThrottling(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Groups[RoleControlPlane] = GroupUsage{ + Containers: 8, ThrottledPeriods: 1, ThrottledSeconds: 0.004, + ThrottledFractionMax: 0.001, ThrottledMaxOf: "ate-api-server", + } + + tr, ok := tripFor(cfg.Check(s), GuardControlPlaneThrottle) + if !ok || !tr.Fatal { + t.Fatalf("a single throttled period in ate-system must trip fatally; got ok=%v trip=%+v", ok, tr) + } +} + +func TestGuardWorkerConnRate(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + // 50,000/s across 100 pods is 500/s each, above the 400/s threshold and + // close to the ~470/s TIME_WAIT ceiling. + s.Envoy.Clusters[ActorClusterName] = ClusterDelta{NewConnectionsPerSec: 50000} + + tr, ok := tripFor(cfg.Check(s), GuardWorkerConnRate) + if !ok || !tr.Fatal { + t.Fatalf("500 new conns/sec per worker pod must trip; got ok=%v trip=%+v", ok, tr) + } + if tr.Value != 500 { + t.Errorf("Value = %v, want 500 (cluster rate divided by pod count)", tr.Value) + } +} + +func TestGuardClientKeepAlive(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Client = ClientStats{NewConnections: 8000, RequestsPerConnection: 1} + + tr, ok := tripFor(cfg.Check(s), GuardClientKeepAlive) + if !ok || !tr.Fatal { + t.Fatalf("a connection per request must trip the keep-alive guard; got ok=%v trip=%+v", ok, tr) + } +} + +func TestGuardClientKeepAliveIgnoresPerfectReuse(t *testing.T) { + // Zero new connections is the healthy steady state, not a divide-by-zero + // worth reporting as a ratio of 0. + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Client = ClientStats{NewConnections: 0, RequestsPerConnection: 0} + + if tr, ok := tripFor(cfg.Check(s), GuardClientKeepAlive); ok { + t.Fatalf("perfect connection reuse tripped the keep-alive guard: %+v", tr) + } +} + +func TestGuardClientKeepAliveIgnoresPreLadderWindow(t *testing.T) { + // The pre-ladder window covers the actor pre-warm: a low ratio by + // construction that says nothing about keep-alive. It tripped a real arm + // at rung 0, so the offered-load floor is a regression test. + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Load.OfferedQPS = 0 + s.Load.AchievedQPS = 0 + s.Client = ClientStats{NewConnections: 32, RequestsPerConnection: 9.53} + + if tr, ok := tripFor(cfg.Check(s), GuardClientKeepAlive); ok { + t.Fatalf("pre-ladder warm-up window tripped the keep-alive guard: %+v", tr) + } + + // The same ratio under real load is still fatal. + s.Load.OfferedQPS = 1000 + s.Load.AchievedQPS = 1000 + if tr, ok := tripFor(cfg.Check(s), GuardClientKeepAlive); !ok || !tr.Fatal { + t.Fatalf("the same ratio under load must still trip; got ok=%v trip=%+v", ok, tr) + } +} + +func TestGuardClientPorts(t *testing.T) { + // The generator dials one router pod IP, so its own source-port range is + // the same 28232 the router has. It would hit that first, and the cliff + // would belong to the rig. + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Client.ConnectionsInUse = int64(cfg.ClientConnectionCeiling) + 1 + + tr, ok := tripFor(cfg.Check(s), GuardClientPorts) + if !ok || !tr.Fatal { + t.Fatalf("exceeding the generator's connection ceiling must trip fatally; got ok=%v trip=%+v", ok, tr) + } + if cfg.ClientConnectionCeiling >= defaultEphemeralPorts { + t.Errorf("ceiling %d leaves no headroom below the %d-port range", cfg.ClientConnectionCeiling, defaultEphemeralPorts) + } +} + +func TestGuardDispatchLagTripsWhenTheSystemIsHealthy(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Load.DispatchLag.P95Ms = 120 + + tr, ok := tripFor(cfg.Check(s), GuardDispatchLag) + if !ok { + t.Fatal("120ms of dispatch lag did not trip the guard") + } + if !tr.Fatal { + t.Error("dispatch lag is fatal when the system is keeping up: the generator, not the router, is the bottleneck") + } +} + +func TestGuardDispatchLagIsNotFatalWhenSaturated(t *testing.T) { + // The exception the ladder depends on. Without it the run would abort at + // precisely the rungs it exists to measure. + cases := []struct { + name string + apply func(*Sample) + }{ + {"LatencySaysSaturated", func(s *Sample) { s.Load.Latency.P95Ms = 400 }}, + {"ThroughputSaysSaturated", func(s *Sample) { s.Load.AchievedQPS = 5000 }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := DefaultGuardConfig() + s := healthySample(&cfg) + s.Load.DispatchLag.P95Ms = 120 + tc.apply(s) + + tr, ok := tripFor(cfg.Check(s), GuardDispatchLag) + if !ok { + t.Fatal("the trip must still be recorded so the reader can see which windows to distrust") + } + if tr.Fatal { + t.Errorf("dispatch lag was fatal under saturation: %s", tr.Detail) + } + if AnyFatal(cfg.Check(s)) { + t.Error("AnyFatal reported true on a saturated window") + } + }) + } +} + +// TestResolveClientCeiling pins the three behaviors: the default ceiling +// follows the pod's measured range, an explicit override survives untouched, +// and a failed read keeps the conservative default rather than guessing. +func TestResolveClientCeiling(t *testing.T) { + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + + g := DefaultGuardConfig() + g.ResolveClientCeiling(func(string) ([]byte, error) { + return []byte("1025\t65535\n"), nil + }, log) + if want := (65535 - 1025 + 1) * 8 / 10; g.ClientConnectionCeiling != want { + t.Errorf("ceiling = %d, want %d (80%% of the widened range)", g.ClientConnectionCeiling, want) + } + + // A flag override is a decision already made; the measured range must not + // undo it. Zero is the strongest override — it disables the guard. + for _, override := range []int{5000, 0} { + g = DefaultGuardConfig() + g.ClientConnectionCeiling = override + g.ResolveClientCeiling(func(string) ([]byte, error) { + return []byte("1025 65535"), nil + }, log) + if g.ClientConnectionCeiling != override { + t.Errorf("override %d was replaced with %d", override, g.ClientConnectionCeiling) + } + } + + g = DefaultGuardConfig() + before := g.ClientConnectionCeiling + g.ResolveClientCeiling(func(string) ([]byte, error) { + return nil, fmt.Errorf("no proc here") + }, log) + if g.ClientConnectionCeiling != before { + t.Errorf("failed read changed the ceiling to %d, want the default %d kept", g.ClientConnectionCeiling, before) + } + + g = DefaultGuardConfig() + g.ResolveClientCeiling(func(string) ([]byte, error) { + return []byte("garbage"), nil + }, log) + if g.ClientConnectionCeiling != before { + t.Errorf("garbage read changed the ceiling to %d, want the default %d kept", g.ClientConnectionCeiling, before) + } +} diff --git a/internal/benchmarking/routercap/kube.go b/internal/benchmarking/routercap/kube.go new file mode 100644 index 000000000..d08b2eafc --- /dev/null +++ b/internal/benchmarking/routercap/kube.go @@ -0,0 +1,287 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Kubernetes wiring: finding the router, worker and generator pods, and building the +// scrape clients that point at them. + +package routercap + +import ( + "context" + "fmt" + "io" + "net/http" + "sort" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// Ports the router pod exposes. Both are container ports on the same pod, so +// the generator reaches them directly at the pod IP — no Service, no +// port-forward, nothing in the path that could itself become a bottleneck. +const ( + envoyAdminPort = 9901 + routerMetricsPort = 9090 +) + +// NewKubeClient builds a clientset. In-cluster first, because that is how the +// run actually executes; the kubeconfig path is for running the binary against +// a cluster from a laptop while developing. +func NewKubeClient(kubeconfig string) (*kubernetes.Clientset, *rest.Config, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + if kubeconfig == "" { + return nil, nil, fmt.Errorf("no in-cluster config and no --kubeconfig: %w", err) + } + cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, nil, fmt.Errorf("load kubeconfig %s: %w", kubeconfig, err) + } + } + cs, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, nil, fmt.Errorf("build kubernetes client: %w", err) + } + return cs, cfg, nil +} + +// PodRef is the subset of a pod the harness needs: where to reach it, which +// node's cAdvisor reports it, and which containers to watch. +type PodRef struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + IP string `json:"ip"` + Node string `json:"node"` + Containers []string `json:"containers"` + // Images maps container name to the image actually running, taken from the + // pod status rather than the spec so a tag resolves to the digest that was + // pulled. + Images map[string]string `json:"images,omitempty"` +} + +// Keys returns a ContainerKey per container in the pod. +func (p PodRef) Keys() []ContainerKey { + out := make([]ContainerKey, 0, len(p.Containers)) + for _, c := range p.Containers { + out = append(out, ContainerKey{Namespace: p.Namespace, Pod: p.Name, Container: c}) + } + return out +} + +// Key returns the ContainerKey for one named container in the pod. +func (p PodRef) Key(container string) ContainerKey { + return ContainerKey{Namespace: p.Namespace, Pod: p.Name, Container: container} +} + +// FindPods lists running, IP-assigned pods matching a label selector, sorted by +// name so repeated calls agree. Pods without an IP or a node are skipped rather +// than returned half-populated: they are mid-startup, and a caller would +// otherwise scrape a blank address. +func FindPods(ctx context.Context, cs kubernetes.Interface, namespace, selector string) ([]PodRef, error) { + list, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, fmt.Errorf("list pods in %s matching %q: %w", namespace, selector, err) + } + var out []PodRef + for i := range list.Items { + p := &list.Items[i] + if p.Status.Phase != corev1.PodRunning || p.Status.PodIP == "" || p.Spec.NodeName == "" { + continue + } + ref := PodRef{Namespace: p.Namespace, Name: p.Name, IP: p.Status.PodIP, Node: p.Spec.NodeName} + for _, c := range p.Spec.Containers { + ref.Containers = append(ref.Containers, c.Name) + } + for _, cs := range p.Status.ContainerStatuses { + if ref.Images == nil { + ref.Images = map[string]string{} + } + // ImageID over Image: the second is whatever the spec asked for, + // the first is the digest the kubelet actually pulled. + img := cs.ImageID + if img == "" { + img = cs.Image + } + ref.Images[cs.Name] = img + } + out = append(out, ref) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +// FindOnePod resolves a selector that must match exactly one pod. Used for the +// router, where two matching pods means a rollout is still in progress and +// measuring either one would blend the old arm with the new. +func FindOnePod(ctx context.Context, cs kubernetes.Interface, namespace, selector string) (PodRef, error) { + pods, err := FindPods(ctx, cs, namespace, selector) + if err != nil { + return PodRef{}, err + } + switch len(pods) { + case 1: + return pods[0], nil + case 0: + return PodRef{}, fmt.Errorf("no running pod in %s matching %q", namespace, selector) + default: + names := make([]string, len(pods)) + for i, p := range pods { + names[i] = p.Name + } + return PodRef{}, fmt.Errorf("%d running pods in %s match %q (%v); a rollout is still in progress and the two would measure as one", + len(pods), namespace, selector, names) + } +} + +// WaitForPod polls until exactly one pod matches, or the context expires. Used +// after an arm's rollout, where the old pod and the new one both exist for a +// while. +func WaitForPod(ctx context.Context, cs kubernetes.Interface, namespace, selector string, poll time.Duration) (PodRef, error) { + pods, err := WaitForPods(ctx, cs, namespace, selector, poll, 1) + if err != nil { + return PodRef{}, err + } + return pods[0], nil +} + +// WaitForPods blocks until the selector matches exactly n running pods, +// returning them ordered by name. Exactly n because during a rollout the +// selector matches old and new pods at once, and measuring that blend labels +// two configurations as one. +func WaitForPods(ctx context.Context, cs kubernetes.Interface, namespace, selector string, poll time.Duration, n int) ([]PodRef, error) { + if poll <= 0 { + poll = 2 * time.Second + } + var last error + for { + pods, err := FindPods(ctx, cs, namespace, selector) + switch { + case err != nil: + last = err + case len(pods) == n: + sort.Slice(pods, func(i, j int) bool { return pods[i].Name < pods[j].Name }) + return pods, nil + default: + last = fmt.Errorf("%d running pods match %q, want %d", len(pods), selector, n) + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for %d pods matching %q in %s: %w (last: %v)", n, selector, namespace, ctx.Err(), last) + case <-time.After(poll): + } + } +} + +// NewCadvisorClient returns a client reading one node's cAdvisor surface +// through the API server's node proxy. That path needs no kubelet client +// certificate — only a ServiceAccount with get on nodes/proxy, which the run's +// RBAC grants. +func NewCadvisorClient(cs kubernetes.Interface, node string) *CadvisorClient { + return &CadvisorClient{ + Fetch: func(ctx context.Context) (io.ReadCloser, error) { + return cs.CoreV1().RESTClient().Get(). + Resource("nodes").Name(node).SubResource("proxy"). + Suffix("metrics", "cadvisor"). + Stream(ctx) + }, + } +} + +// NewMultiNodeCadvisorClient returns a client covering every node the run +// watches containers on. Nodes are deduplicated, so callers can pass one node +// name per pod without caring how the pods were scheduled. +func NewMultiNodeCadvisorClient(cs kubernetes.Interface, nodes []string) *MultiNodeClient { + m := &MultiNodeClient{} + seen := map[string]bool{} + for _, n := range nodes { + if n == "" || seen[n] { + continue + } + seen[n] = true + m.Clients = append(m.Clients, NewCadvisorClient(cs, n)) + } + return m +} + +// NewScrapeHTTPClient returns the shared client for the router pod's two +// Prometheus endpoints. Exported so the binary can give both scrapers one +// connection pool instead of two. +func NewScrapeHTTPClient() *http.Client { return newScrapeHTTPClient() } + +// scrapeHTTP is the fetcher for the two in-pod Prometheus endpoints. It holds +// its own http.Client with keep-alive so scraping does not itself churn +// connections while the run is measuring connection churn. +func scrapeHTTP(client *http.Client, url string) func(ctx context.Context) (io.ReadCloser, error) { + return func(ctx context.Context) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + return nil, fmt.Errorf("GET %s returned %d: %s", url, resp.StatusCode, body) + } + return resp.Body, nil + } +} + +// newScrapeHTTPClient returns the client used for the two metrics endpoints. +// Short timeouts: a scrape that has not answered within a few seconds has +// already missed the window it belongs to, and hanging on it would stall the +// sampler that drives the whole series. +func newScrapeHTTPClient() *http.Client { + return &http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 90 * time.Second, + }, + } +} + +func envoyStatsURL(pod PodRef) string { + return fmt.Sprintf("http://%s:%d/stats/prometheus", pod.IP, envoyAdminPort) +} + +func routerStatsURL(pod PodRef) string { + return fmt.Sprintf("http://%s:%d/metrics", pod.IP, routerMetricsPort) +} + +// NewEnvoyClient returns a client for the router pod's Envoy admin endpoint. +func NewEnvoyClient(hc *http.Client, pod PodRef) *EnvoyClient { + return &EnvoyClient{Fetch: scrapeHTTP(hc, envoyStatsURL(pod))} +} + +// NewContentionClient returns a client for the same admin server's /contention +// endpoint, which carries mutex-contention counters when Envoy runs with +// --enable-mutex-tracing. +func NewContentionClient(hc *http.Client, pod PodRef) *ContentionClient { + return &ContentionClient{Fetch: scrapeHTTP(hc, + fmt.Sprintf("http://%s:%d/contention", pod.IP, envoyAdminPort))} +} + +// NewRouterClient returns a client for the sidecar's own metrics endpoint. +func NewRouterClient(hc *http.Client, pod PodRef) *RouterClient { + return &RouterClient{Fetch: scrapeHTTP(hc, routerStatsURL(pod))} +} diff --git a/internal/benchmarking/routercap/kube_test.go b/internal/benchmarking/routercap/kube_test.go new file mode 100644 index 000000000..bc59db60d --- /dev/null +++ b/internal/benchmarking/routercap/kube_test.go @@ -0,0 +1,223 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for pod discovery and for the endpoints the scrape clients target. + +package routercap + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func pod(name, ip, node string, phase corev1.PodPhase, containers ...string) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ate-system", Name: name, Labels: map[string]string{"app": "atenet-router"}}, + Status: corev1.PodStatus{Phase: phase, PodIP: ip}, + Spec: corev1.PodSpec{NodeName: node}, + } + for _, c := range containers { + p.Spec.Containers = append(p.Spec.Containers, corev1.Container{Name: c}) + } + return p +} + +func TestFindPodsSkipsPodsThatCannotBeScraped(t *testing.T) { + cs := fake.NewSimpleClientset( + pod("router-b", "10.0.0.2", "n1", corev1.PodRunning, "envoy", "atenet-router"), + pod("router-a", "10.0.0.1", "n1", corev1.PodRunning, "envoy", "atenet-router"), + // Mid-startup: no IP yet. Returning it would give the scraper a blank + // address to dial. + pod("router-c", "", "n1", corev1.PodPending, "envoy"), + // Terminated: still listed by the API, but has no live metrics. + pod("router-d", "10.0.0.4", "n1", corev1.PodSucceeded, "envoy"), + ) + + got, err := FindPods(context.Background(), cs, "ate-system", "app=atenet-router") + if err != nil { + t.Fatalf("FindPods: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d pods, want 2: %+v", len(got), got) + } + // Sorted, so a re-resolve mid-run does not silently reorder the targets. + if got[0].Name != "router-a" || got[1].Name != "router-b" { + t.Errorf("pods not sorted by name: %v, %v", got[0].Name, got[1].Name) + } + if got[0].Node != "n1" || got[0].IP != "10.0.0.1" { + t.Errorf("pod = %+v, want node n1 at 10.0.0.1", got[0]) + } + keys := got[0].Keys() + if len(keys) != 2 || keys[0] != (ContainerKey{"ate-system", "router-a", "envoy"}) { + t.Errorf("Keys() = %+v, want one per container", keys) + } +} + +func TestFindOnePodRefusesAnInProgressRollout(t *testing.T) { + // Two running router pods means the old arm and the new one are both live. + // Picking either would blend them, so this has to be an error. + cs := fake.NewSimpleClientset( + pod("router-old", "10.0.0.1", "n1", corev1.PodRunning, "envoy"), + pod("router-new", "10.0.0.2", "n1", corev1.PodRunning, "envoy"), + ) + + _, err := FindOnePod(context.Background(), cs, "ate-system", "app=atenet-router") + if err == nil { + t.Fatal("FindOnePod accepted two matching pods") + } + if !strings.Contains(err.Error(), "rollout") { + t.Errorf("error = %q, want it to name the rollout", err) + } +} + +func TestFindOnePodErrorsWhenNothingMatches(t *testing.T) { + cs := fake.NewSimpleClientset() + if _, err := FindOnePod(context.Background(), cs, "ate-system", "app=atenet-router"); err == nil { + t.Fatal("FindOnePod returned no error with no pods") + } +} + +func TestWaitForPodGivesUpWithTheUnderlyingReason(t *testing.T) { + cs := fake.NewSimpleClientset() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err := WaitForPod(ctx, cs, "ate-system", "app=atenet-router", 10*time.Millisecond) + if err == nil { + t.Fatal("WaitForPod returned no error after its context expired") + } + // The deadline alone would not say what was being waited for. + if !strings.Contains(err.Error(), "0 running pods match") { + t.Errorf("error = %q, want it to carry the last underlying reason", err) + } +} + +// TestWaitForPodsDemandsTheExactCount pins the replica-aware wait: fewer pods +// than asked for is a rollout in progress, not a smaller success, and the pods +// come back in name order so replica 0 is stable across calls. +func TestWaitForPodsDemandsTheExactCount(t *testing.T) { + cs := fake.NewSimpleClientset( + pod("router-b", "10.0.0.2", "n2", corev1.PodRunning, "envoy"), + pod("router-a", "10.0.0.1", "n1", corev1.PodRunning, "envoy"), + ) + + pods, err := WaitForPods(context.Background(), cs, "ate-system", "", 10*time.Millisecond, 2) + if err != nil { + t.Fatalf("WaitForPods: %v", err) + } + if pods[0].Name != "router-a" || pods[1].Name != "router-b" { + t.Errorf("pods = %s,%s, want name order router-a,router-b", pods[0].Name, pods[1].Name) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := WaitForPods(ctx, cs, "ate-system", "", 10*time.Millisecond, 3); err == nil { + t.Fatal("WaitForPods accepted 2 pods when 3 were demanded") + } else if !strings.Contains(err.Error(), "2 running pods match") { + t.Errorf("error = %q, want it to name the shortfall", err) + } +} + +func TestScrapeHTTPSurfacesTheBodyOnAnErrorStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "admin endpoint disabled", http.StatusForbidden) + })) + defer srv.Close() + + _, err := scrapeHTTP(srv.Client(), srv.URL)(context.Background()) + if err == nil { + t.Fatal("a 403 did not produce an error") + } + if !strings.Contains(err.Error(), "403") || !strings.Contains(err.Error(), "admin endpoint disabled") { + t.Errorf("error = %q, want both the status and the body", err) + } +} + +func TestEnvoyAndRouterClientsTargetThePodDirectly(t *testing.T) { + var paths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + io.WriteString(w, envoyFixture(1, 1, 1)) + })) + defer srv.Close() + + // The real clients build a pod-IP URL; point them at the test server by + // substituting the fetcher, and assert the path each one asks for. + hc := srv.Client() + e := &EnvoyClient{Fetch: scrapeHTTP(hc, srv.URL+"/stats/prometheus")} + if _, err := e.Scrape(context.Background()); err != nil { + t.Fatalf("envoy scrape: %v", err) + } + r := &RouterClient{Fetch: scrapeHTTP(hc, srv.URL+"/metrics")} + if _, err := r.Scrape(context.Background()); err != nil { + t.Fatalf("router scrape: %v", err) + } + + want := []string{"/stats/prometheus", "/metrics"} + if len(paths) != 2 || paths[0] != want[0] || paths[1] != want[1] { + t.Errorf("paths = %v, want %v", paths, want) + } + + // And the URLs the constructors actually build, so a port typo is caught + // here rather than as an empty series at run time. + p := PodRef{Namespace: "ate-system", Name: "router-a", IP: "10.0.0.1", Node: "n1"} + if got := envoyStatsURL(p); got != "http://10.0.0.1:9901/stats/prometheus" { + t.Errorf("envoy URL = %q", got) + } + if got := routerStatsURL(p); got != "http://10.0.0.1:9090/metrics" { + t.Errorf("router URL = %q", got) + } +} + +func TestFindPodsRecordsTheImageActuallyRunning(t *testing.T) { + // Provenance: which build produced the numbers. Taken from status, not + // spec, so a mutable tag resolves to the digest that was pulled. + p := pod("router-a", "10.0.0.1", "n1", corev1.PodRunning, "envoy", "atenet-router") + p.Status.ContainerStatuses = []corev1.ContainerStatus{ + {Name: "envoy", Image: "envoy:v1.35", ImageID: "docker.io/envoyproxy/envoy@sha256:abc"}, + // No ImageID yet: the spec's tag is better than nothing. + {Name: "atenet-router", Image: "ko://atenet@latest"}, + } + cs := fake.NewSimpleClientset(p) + + got, err := FindPods(context.Background(), cs, "ate-system", "app=atenet-router") + if err != nil { + t.Fatalf("FindPods: %v", err) + } + if want := "docker.io/envoyproxy/envoy@sha256:abc"; got[0].Images["envoy"] != want { + t.Errorf("envoy image = %q, want the pulled digest %q", got[0].Images["envoy"], want) + } + if want := "ko://atenet@latest"; got[0].Images["atenet-router"] != want { + t.Errorf("sidecar image = %q, want the spec image as a fallback %q", got[0].Images["atenet-router"], want) + } +} + +func TestNewMultiNodeCadvisorClientDeduplicatesNodes(t *testing.T) { + // Callers pass one node name per pod; a hundred worker pods on two nodes + // must not become a hundred scrapes of the same kubelet every window. + cs := fake.NewSimpleClientset() + m := NewMultiNodeCadvisorClient(cs, []string{"n1", "n2", "n1", "", "n2"}) + if len(m.Clients) != 2 { + t.Errorf("built %d clients for 2 distinct nodes", len(m.Clients)) + } +} diff --git a/internal/benchmarking/routercap/output.go b/internal/benchmarking/routercap/output.go new file mode 100644 index 000000000..d12046efe --- /dev/null +++ b/internal/benchmarking/routercap/output.go @@ -0,0 +1,322 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Where records go — JSONL files for a local run, tagged stdout for the in-cluster +// Job — and the ladder spec that expands into the rungs a run walks. + +package routercap + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" +) + +// Output file names. Fixed rather than configurable so charts.py and any future +// automation can find them without being told. +const ( + SamplesFile = "samples.jsonl" + FineFile = "fine.jsonl" + HeaderFile = "run.json" +) + +// Sink receives the run's records. Two streams rather than one because the two +// series have different resolutions and only one of them is aligned to the +// resource panels — see the comment on FineSample. +type Sink interface { + Sample(Sample) error + Fine(FineSample) error +} + +// JSONLSink writes newline-delimited JSON to two writers. JSONL and unbuffered +// on purpose: a killed run still leaves every completed line intact and +// plottable. +type JSONLSink struct { + mu sync.Mutex + samples *json.Encoder + fine *json.Encoder + closers []func() error +} + +// NewJSONLSink writes to the given writers. A nil writer disables that stream. +func NewJSONLSink(samples, fine io.Writer) *JSONLSink { + s := &JSONLSink{} + if samples != nil { + s.samples = json.NewEncoder(samples) + } + if fine != nil { + s.fine = json.NewEncoder(fine) + } + return s +} + +// OpenJSONLSink creates dir and the two output files inside it. +func OpenJSONLSink(dir string) (*JSONLSink, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create output dir %s: %w", dir, err) + } + sf, err := os.Create(filepath.Join(dir, SamplesFile)) + if err != nil { + return nil, err + } + ff, err := os.Create(filepath.Join(dir, FineFile)) + if err != nil { + sf.Close() + return nil, err + } + s := NewJSONLSink(sf, ff) + s.closers = []func() error{sf.Close, ff.Close} + return s, nil +} + +// Sample writes one aligned record. +func (s *JSONLSink) Sample(v Sample) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.samples == nil { + return nil + } + return s.samples.Encode(v) +} + +// Fine writes one generator-only record. +func (s *JSONLSink) Fine(v FineSample) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.fine == nil { + return nil + } + return s.fine.Encode(v) +} + +// Close releases the underlying files. +func (s *JSONLSink) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + var first error + for _, c := range s.closers { + if err := c(); err != nil && first == nil { + first = err + } + } + s.closers = nil + return first +} + +// Stream names used when both record series share one writer. +const ( + StreamSample = "sample" + StreamFine = "fine" + StreamHeader = "header" +) + +// streamLine tags one record with which stream it belongs to. +type streamLine struct { + Stream string `json:"stream"` + Record any `json:"record"` +} + +// StreamSink writes both series and the header, tagged, to a single writer. +// The generator is a Job in a distroless container, so stdout is the only +// channel that survives; run.sh splits the tagged lines back into the files a +// laptop run writes directly. +type StreamSink struct { + mu sync.Mutex + enc *json.Encoder +} + +// NewStreamSink writes tagged JSONL to w. +func NewStreamSink(w io.Writer) *StreamSink { + return &StreamSink{enc: json.NewEncoder(w)} +} + +// Sample writes one aligned record. +func (s *StreamSink) Sample(v Sample) error { return s.write(StreamSample, v) } + +// Fine writes one generator-only record. +func (s *StreamSink) Fine(v FineSample) error { return s.write(StreamFine, v) } + +// Header writes the run header as a tagged line. +func (s *StreamSink) Header(v RunHeader) error { return s.write(StreamHeader, v) } + +func (s *StreamSink) write(stream string, v any) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.enc.Encode(streamLine{Stream: stream, Record: v}) +} + +// MultiSink fans each record out to every sink, and reports every failure +// rather than the first. A local file failing to write is not a reason to stop +// streaming the same record to stdout, which may be the copy that survives. +type MultiSink []Sink + +// Sample writes one aligned record to every sink. +func (m MultiSink) Sample(v Sample) error { + errs := make([]error, 0, len(m)) + for _, s := range m { + errs = append(errs, s.Sample(v)) + } + return errors.Join(errs...) +} + +// Fine writes one generator-only record to every sink. +func (m MultiSink) Fine(v FineSample) error { + errs := make([]error, 0, len(m)) + for _, s := range m { + errs = append(errs, s.Fine(v)) + } + return errors.Join(errs...) +} + +// LadderSpec describes one sweep of offered load. Held as a spec rather than as +// a materialized slice so the run header can record what was asked for in four +// numbers instead of sixteen rung objects. +type LadderSpec struct { + StartQPS float64 `json:"start_qps"` + StepQPS float64 `json:"step_qps"` + Rungs int `json:"rungs"` + // Hold is how long each rung runs, and Warmup the leading part of it + // excluded from the summary. Warmup samples are still written. + Hold time.Duration `json:"hold"` + Warmup time.Duration `json:"warmup"` +} + +// Build materializes the rungs. StartAt is left zero: the pacer stamps it when +// the rung actually begins, since that depends on how long the previous rung +// took to finish dispatching. +func (l LadderSpec) Build() []Rung { + out := make([]Rung, 0, l.Rungs) + for i := 0; i < l.Rungs; i++ { + out = append(out, Rung{ + Index: i, + RateQPS: l.StartQPS + float64(i)*l.StepQPS, + Hold: l.Hold, + Warmup: l.Warmup, + }) + } + return out +} + +// PeakQPS is the top rung's rate, which is what the in-flight cap and the +// connection pool have to be sized against. +func (l LadderSpec) PeakQPS() float64 { + if l.Rungs <= 0 { + return 0 + } + return l.StartQPS + float64(l.Rungs-1)*l.StepQPS +} + +// PortRange is the router pod's ephemeral source-port range. Read out of the +// live pod rather than assumed, because every claim about the port wall is a +// claim about these two numbers, and Source records whether the read succeeded. +type PortRange struct { + Low int `json:"low"` + High int `json:"high"` + // Source is "measured" when read from the router pod's + // net.ipv4.ip_local_port_range, "assumed" when the read failed and the + // Linux default was substituted. + Source string `json:"source"` +} + +const ( + PortRangeMeasured = "measured" + PortRangeAssumed = "assumed" +) + +// DefaultPortRange is the Linux default, used only when the live read fails. +func DefaultPortRange() PortRange { + return PortRange{Low: 32768, High: 60999, Source: PortRangeAssumed} +} + +// Size is the number of ports in the range. +func (p PortRange) Size() int { + if p.High < p.Low || p.Low <= 0 { + return 0 + } + return p.High - p.Low + 1 +} + +// RunHeader is run.json: everything needed to know what experiment produced the +// samples sitting beside it. A chart six months from now should not need this +// conversation, a git log, or a cluster that still exists. +type RunHeader struct { + Name string `json:"name"` + Tag string `json:"tag,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at,omitempty"` + + GitSHA string `json:"git_sha,omitempty"` + RouterImage string `json:"router_image,omitempty"` + + Cluster string `json:"cluster,omitempty"` + Location string `json:"location,omitempty"` + MachineType string `json:"machine_type,omitempty"` + + RouterPod PodRef `json:"router_pod"` + // RouterPods lists every replica when the run drove more than one + // (-router-pods). RouterPod is then the anchor — the pod the Envoy and + // sidecar scrapes describe, which saw 1/len(RouterPods) of the traffic. + RouterPods []PodRef `json:"router_pods,omitempty"` + // Placement maps role to node name. A run where the generator landed on the + // router's node is a different experiment from one where it did not, and + // the taints that prevent it can be removed by anyone with kubectl. + Placement map[string]string `json:"node_placement,omitempty"` + + PortRange PortRange `json:"port_range"` + // CircuitBreakerLimit and ExtProcMaxRequests are recorded because the + // ordering claim — Envoy's counted overflow trips before the kernel's + // opaque EADDRNOTAVAIL — holds only if both sit below PortRange.Size(). + CircuitBreakerLimit int `json:"circuit_breaker_limit"` + ExtProcMaxRequests int `json:"extproc_max_requests,omitempty"` + + ArmCores []int `json:"arm_cores"` + Actors int `json:"actors"` + Ladder LadderSpec `json:"ladder"` + Guards GuardConfig `json:"guards"` + + Results []RunResult `json:"results,omitempty"` + Caveats []string `json:"caveats"` +} + +// WriteHeader writes run.json into dir. +func WriteHeader(dir string, h RunHeader) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create output dir %s: %w", dir, err) + } + b, err := json.MarshalIndent(h, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, HeaderFile), append(b, '\n'), 0o644) +} + +// StandingCaveats are the qualifications a reader needs to read the charts +// correctly, true of every run; they ship inside run.json so a chart read +// later carries its own fine print. The bar for the list: it must change how +// a number on the report should be read. +func StandingCaveats() []string { + return []string{ + "Container CPU and memory come from cAdvisor on the kubelet, whose ~10s housekeeping cadence sets the width of every window on these charts. That is the real resolution of any container CPU number on a kubelet-managed node.", + "CPU in a window is the mean over that window, so a burst shorter than the window is invisible in the CPU panel even when it is plainly visible in the latency panel.", + "Latency is measured from each request's scheduled send time, not from when it reached the wire, so client-side queueing is inside the number and coordinated omission is not possible.", + "Failures and timeouts contribute their full latency to the percentiles rather than being dropped from them.", + "Offered QPS is read from the pacer's fixed schedule, not counted from what the generator emitted, so a struggling generator cannot quietly redefine the x-axis.", + "The generator's connection pool is not a setting: its transport dials without a per-host cap, so one stalled second makes every blocked request open its own connection and the pool steps up by thousands, then holds that size for the 120s idle timeout. A step in the pool series followed by a latency hump at unchanged offered load is the pool re-settling, not the router's capacity.", + } +} diff --git a/internal/benchmarking/routercap/pacer.go b/internal/benchmarking/routercap/pacer.go new file mode 100644 index 000000000..5f0ebd554 --- /dev/null +++ b/internal/benchmarking/routercap/pacer.go @@ -0,0 +1,208 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The open-loop pacer and the rung schedule it walks. Open-loop: it fires on a fixed +// tick whether or not earlier requests have come back. + +package routercap + +import ( + "context" + "sync" + "time" +) + +// Rung is one step of the ladder: a constant offered rate held for a fixed +// duration. +type Rung struct { + Index int `json:"index"` + RateQPS float64 `json:"rate_qps"` + Hold time.Duration `json:"hold"` + // Warmup is the leading part of the rung excluded from the summary; the + // samples are still written. + Warmup time.Duration `json:"warmup"` + // StartAt is set when the rung actually begins, since it depends on how + // long the preceding rung's teardown took. + StartAt time.Time `json:"start_at"` +} + +// End is the instant after which the pacer schedules nothing more for r. +func (r Rung) End() time.Time { return r.StartAt.Add(r.Hold) } + +// Schedule is the record of what the pacer was asked to produce. Offered load +// is read from here rather than counted, so a struggling generator cannot +// quietly redefine the x-axis. +type Schedule struct { + mu sync.RWMutex + rungs []Rung +} + +// Begin appends r as started, and returns it with StartAt filled in. +func (s *Schedule) Begin(r Rung, at time.Time) Rung { + r.StartAt = at + s.mu.Lock() + s.rungs = append(s.rungs, r) + s.mu.Unlock() + return r +} + +// Rungs returns a copy of the rungs begun so far. +func (s *Schedule) Rungs() []Rung { + s.mu.RLock() + defer s.mu.RUnlock() + return append([]Rung(nil), s.rungs...) +} + +// OfferedIn is how many requests the schedule placed in [t0, t1). Intervals +// that straddle a rung boundary get each rung's share, so a window is never +// attributed a rate the pacer was not actually running. +func (s *Schedule) OfferedIn(t0, t1 time.Time) float64 { + s.mu.RLock() + defer s.mu.RUnlock() + var total float64 + for _, r := range s.rungs { + lo, hi := r.StartAt, r.End() + if t0.After(lo) { + lo = t0 + } + if t1.Before(hi) { + hi = t1 + } + if d := hi.Sub(lo); d > 0 { + total += r.RateQPS * d.Seconds() + } + } + return total +} + +// RungAt returns the rung covering t, and whether t falls inside that rung's +// warmup prefix. ok is false for instants between rungs. +func (s *Schedule) RungAt(t time.Time) (r Rung, warmup, ok bool) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, cand := range s.rungs { + if t.Before(cand.StartAt) || !t.Before(cand.End()) { + continue + } + return cand, t.Before(cand.StartAt.Add(cand.Warmup)), true + } + return Rung{}, false, false +} + +// SendFunc performs one request and classifies it. It must not retry: a retry +// hidden inside the send would show up as one slow request instead of two, and +// the retry's own load would be invisible to the offered rate. +type SendFunc func(ctx context.Context) (Outcome, int) + +// Pacer drives an open loop: it emits requests on a schedule fixed in advance +// and lets concurrency grow as the system slows. Latency is measured from each +// request's scheduled send time, which rules out coordinated omission. +type Pacer struct { + Collector *Collector + // MaxInFlight bounds the generator's own concurrency; reaching it is a rig + // failure, not a result. Requests beyond it are recorded as shed and the + // guards trip. + MaxInFlight int64 + // TickCap bounds how long the dispatch loop sleeps, and so bounds the + // dispatch lag the loop itself can introduce. At rates where the + // inter-arrival gap is below the OS timer granularity the loop wakes on + // this interval and emits the whole batch that has come due. + TickCap time.Duration +} + +// RunRung emits r's requests on schedule and returns once the last one has +// been dispatched. It does not wait for in-flight requests to complete — +// draining at a rung boundary would idle the system between rungs and make the +// next rung's first seconds measure a cold pool rather than a running one. +func (p *Pacer) RunRung(ctx context.Context, r Rung, send SendFunc) error { + if r.RateQPS <= 0 || r.Hold <= 0 { + return nil + } + tickCap := p.TickCap + if tickCap <= 0 { + tickCap = time.Millisecond + } + + total := int(r.RateQPS * r.Hold.Seconds()) + gap := float64(time.Second) / r.RateQPS + at := func(i int) time.Time { + // Offsets are computed from the rung start rather than accumulated, so + // a late wake-up delays one request instead of every request after it. + return r.StartAt.Add(time.Duration(float64(i) * gap)) + } + + timer := time.NewTimer(time.Hour) + defer timer.Stop() + + for i := 0; i < total; { + now := time.Now() + for i < total && !at(i).After(now) { + p.fire(ctx, at(i), send) + i++ + } + if i >= total { + break + } + wait := time.Until(at(i)) + if wait > tickCap { + wait = tickCap + } + if wait <= 0 { + continue + } + timer.Reset(wait) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return ctx.Err() + case <-timer.C: + } + } + return ctx.Err() +} + +// fire launches one request. The goroutine-per-request shape is what makes the +// loop open: the dispatcher never blocks on a response, so a slow system grows +// the in-flight count instead of throttling the offered rate. +func (p *Pacer) fire(ctx context.Context, scheduled time.Time, send SendFunc) { + if p.MaxInFlight > 0 && p.Collector.InFlight() >= p.MaxInFlight { + p.Collector.RecordShed(scheduled, time.Now()) + return + } + go func() { + p.Collector.RecordDispatch(scheduled, time.Now()) + outcome, status := send(ctx) + p.Collector.RecordCompletion(scheduled, time.Now(), outcome, status) + }() +} + +// Drain blocks until nothing is in flight or timeout elapses, reporting +// whether it emptied. Used between arms, where the next arm restarts the +// router pod and any request still outstanding would be attributed to it. +func (p *Pacer) Drain(ctx context.Context, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if p.Collector.InFlight() == 0 { + return true + } + select { + case <-ctx.Done(): + return p.Collector.InFlight() == 0 + case <-time.After(20 * time.Millisecond): + } + } + return p.Collector.InFlight() == 0 +} diff --git a/internal/benchmarking/routercap/pacer_test.go b/internal/benchmarking/routercap/pacer_test.go new file mode 100644 index 000000000..2dfbc99a3 --- /dev/null +++ b/internal/benchmarking/routercap/pacer_test.go @@ -0,0 +1,374 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for the pacer, the rung schedule, and the collector's windowed statistics. + +package routercap + +import ( + "context" + "math" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestSummarize(t *testing.T) { + t.Run("NearestRankPicksAnObservedValue", func(t *testing.T) { + // 1..100ms. Nearest rank puts p50 at the 50th value and p95 at the + // 95th, both of which are latencies that actually occurred. + var ds []time.Duration + for i := 1; i <= 100; i++ { + ds = append(ds, time.Duration(i)*time.Millisecond) + } + got := summarize(ds) + for _, tc := range []struct { + name string + got float64 + want float64 + }{ + {"count", float64(got.Count), 100}, + {"p50", got.P50Ms, 50}, + {"p95", got.P95Ms, 95}, + {"mean", got.MeanMs, 50.5}, + } { + if math.Abs(tc.got-tc.want) > 1e-9 { + t.Errorf("%s = %v, want %v", tc.name, tc.got, tc.want) + } + } + }) + + t.Run("UnsortedInputIsSorted", func(t *testing.T) { + // Nearest rank puts p95 of three samples at the largest. Unsorted, the + // same index would land on the 5ms sample instead. + ds := []time.Duration{9 * time.Millisecond, time.Millisecond, 5 * time.Millisecond} + if got := summarize(ds).P95Ms; got != 9 { + t.Errorf("p95 = %v, want 9", got) + } + }) + + t.Run("EmptyIsZeroNotPanic", func(t *testing.T) { + if got := summarize(nil); got.Count != 0 || got.P95Ms != 0 { + t.Errorf("summarize(nil) = %+v, want zero", got) + } + }) + + t.Run("SingleSampleIsEveryPercentile", func(t *testing.T) { + got := summarize([]time.Duration{7 * time.Millisecond}) + if got.P50Ms != 7 || got.P95Ms != 7 || got.MeanMs != 7 { + t.Errorf("single-sample percentiles = %+v, want all 7ms", got) + } + }) +} + +func TestScheduleOfferedIn(t *testing.T) { + base := time.Unix(1_800_000_000, 0) + s := &Schedule{} + s.Begin(Rung{Index: 0, RateQPS: 1000, Hold: 10 * time.Second}, base) + s.Begin(Rung{Index: 1, RateQPS: 2000, Hold: 10 * time.Second}, base.Add(10*time.Second)) + + tests := []struct { + name string + t0, t1 time.Time + want float64 + }{ + {"WhollyInsideFirstRung", base, base.Add(5 * time.Second), 5000}, + {"WhollyInsideSecondRung", base.Add(12 * time.Second), base.Add(15 * time.Second), 6000}, + // The case that matters: a window driven by cAdvisor's clock does not + // respect rung boundaries, so it must be credited each rung's share. + {"StraddlesTheBoundary", base.Add(8 * time.Second), base.Add(13 * time.Second), 2000 + 6000}, + {"BeforeAnyRung", base.Add(-5 * time.Second), base, 0}, + {"AfterEveryRung", base.Add(20 * time.Second), base.Add(25 * time.Second), 0}, + {"SpansEverything", base.Add(-1 * time.Second), base.Add(21 * time.Second), 30000}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := s.OfferedIn(tc.t0, tc.t1); math.Abs(got-tc.want) > 1e-6 { + t.Errorf("OfferedIn = %v, want %v", got, tc.want) + } + }) + } +} + +func TestScheduleRungAt(t *testing.T) { + base := time.Unix(1_800_000_000, 0) + s := &Schedule{} + s.Begin(Rung{Index: 3, RateQPS: 500, Hold: 45 * time.Second, Warmup: 10 * time.Second}, base) + + for _, tc := range []struct { + name string + at time.Time + wantOK bool + wantWarmup bool + }{ + {"AtStartIsWarmup", base, true, true}, + {"JustBeforeWarmupEnds", base.Add(9 * time.Second), true, true}, + {"AtWarmupEndIsMeasured", base.Add(10 * time.Second), true, false}, + {"LateInRungIsMeasured", base.Add(44 * time.Second), true, false}, + {"AtRungEndIsOutside", base.Add(45 * time.Second), false, false}, + {"BeforeRungIsOutside", base.Add(-time.Second), false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + r, warmup, ok := s.RungAt(tc.at) + if ok != tc.wantOK { + t.Fatalf("ok = %v, want %v", ok, tc.wantOK) + } + if ok && r.Index != 3 { + t.Errorf("rung index = %d, want 3", r.Index) + } + if warmup != tc.wantWarmup { + t.Errorf("warmup = %v, want %v", warmup, tc.wantWarmup) + } + }) + } +} + +func TestCollectorStats(t *testing.T) { + base := time.Unix(1_800_000_000, 0) + sched := &Schedule{} + sched.Begin(Rung{RateQPS: 100, Hold: 10 * time.Second}, base) + c := NewCollector(sched) + + // Three requests completing inside [base+1s, base+2s), each scheduled 1s + // before it completed, plus one that completes after the window. + for i, off := range []time.Duration{1100, 1200, 1300} { + at := base.Add(off * time.Millisecond) + c.RecordDispatch(at.Add(-time.Second), at.Add(-time.Second)) + c.RecordCompletion(at.Add(-time.Second), at, OutcomeOK, 200) + _ = i + } + late := base.Add(5 * time.Second) + c.RecordDispatch(late, late) + c.RecordCompletion(late, late.Add(time.Second), OutcomeHTTPError, 503) + + got := c.Stats(base.Add(time.Second), base.Add(2*time.Second)) + + if got.OfferedQPS != 100 { + t.Errorf("offered = %v, want 100 (from the schedule, not from what completed)", got.OfferedQPS) + } + if got.AchievedQPS != 3 { + t.Errorf("achieved = %v, want 3", got.AchievedQPS) + } + if got.SuccessQPS != 3 { + t.Errorf("success = %v, want 3", got.SuccessQPS) + } + // Every one of the three waited exactly a second from its scheduled time. + if got.Latency.Count != 3 || math.Abs(got.Latency.P50Ms-1000) > 1 { + t.Errorf("latency = %+v, want 3 samples at ~1000ms", got.Latency) + } + if got.Outcomes[OutcomeHTTPError] != 0 { + t.Errorf("the 503 completed outside the window but was counted: %+v", got.Outcomes) + } +} + +func TestCollectorLatencyIsFromScheduledTime(t *testing.T) { + // A request dispatched 400ms late and answered 100ms later took 500ms from + // the caller's view. Reporting 100ms would be coordinated omission. + base := time.Unix(1_800_000_000, 0) + sched := &Schedule{} + sched.Begin(Rung{RateQPS: 1, Hold: time.Minute}, base) + c := NewCollector(sched) + + scheduled := base + dispatched := base.Add(400 * time.Millisecond) + completed := dispatched.Add(100 * time.Millisecond) + c.RecordDispatch(scheduled, dispatched) + c.RecordCompletion(scheduled, completed, OutcomeOK, 200) + + got := c.Stats(base, base.Add(time.Second)) + if math.Abs(got.Latency.P50Ms-500) > 1 { + t.Errorf("latency p50 = %vms, want 500ms measured from the scheduled send time", got.Latency.P50Ms) + } + if math.Abs(got.DispatchLag.P50Ms-400) > 1 { + t.Errorf("dispatch lag p50 = %vms, want 400ms", got.DispatchLag.P50Ms) + } +} + +func TestCollectorDispatchLagIncludesUnfinishedRequests(t *testing.T) { + // A request dispatched inside the window but still in flight when it + // closes must still contribute its lag. Waiting for completions would drop + // exactly the requests the system is struggling with. + base := time.Unix(1_800_000_000, 0) + sched := &Schedule{} + sched.Begin(Rung{RateQPS: 1, Hold: time.Minute}, base) + c := NewCollector(sched) + + c.RecordDispatch(base, base.Add(250*time.Millisecond)) + + got := c.Stats(base, base.Add(time.Second)) + if got.DispatchLag.Count != 1 { + t.Fatalf("dispatch lag count = %d, want 1 for an in-flight request", got.DispatchLag.Count) + } + if got.AchievedQPS != 0 { + t.Errorf("achieved = %v, want 0 — nothing completed yet", got.AchievedQPS) + } + if got.InFlightEnd != 1 { + t.Errorf("in-flight = %d, want 1", got.InFlightEnd) + } +} + +func TestCollectorShedIsExcludedFromLatency(t *testing.T) { + // A shed request never reached the server, so it has no server latency to + // contribute. Counting a zero would drag the reported percentiles down at + // precisely the moment the rig was failing. + base := time.Unix(1_800_000_000, 0) + sched := &Schedule{} + sched.Begin(Rung{RateQPS: 10, Hold: time.Minute}, base) + c := NewCollector(sched) + + c.RecordDispatch(base, base) + c.RecordCompletion(base, base.Add(900*time.Millisecond), OutcomeOK, 200) + for i := 0; i < 5; i++ { + c.RecordShed(base, base.Add(100*time.Millisecond)) + } + + got := c.Stats(base, base.Add(time.Second)) + if got.Latency.Count != 1 { + t.Errorf("latency count = %d, want 1 (shed requests excluded)", got.Latency.Count) + } + if math.Abs(got.Latency.P50Ms-900) > 1 { + t.Errorf("latency p50 = %v, want 900ms undiluted by shed zeroes", got.Latency.P50Ms) + } + if got.Outcomes[OutcomeShed] != 5 { + t.Errorf("shed count = %d, want 5 — shed must still be visible", got.Outcomes[OutcomeShed]) + } + if got.AchievedQPS != 6 { + t.Errorf("achieved = %v, want 6: shed requests are accounted for, just not timed", got.AchievedQPS) + } +} + +func TestCollectorPrune(t *testing.T) { + base := time.Unix(1_800_000_000, 0) + sched := &Schedule{} + sched.Begin(Rung{RateQPS: 1, Hold: time.Minute}, base) + c := NewCollector(sched) + + for i := 0; i < 10; i++ { + at := base.Add(time.Duration(i) * time.Second) + c.RecordDispatch(at, at) + c.RecordCompletion(at, at, OutcomeOK, 200) + } + c.Prune(base.Add(5 * time.Second)) + + if got := c.Stats(base, base.Add(5*time.Second)); got.AchievedQPS != 0 { + t.Errorf("pruned interval still reports %v achieved", got.AchievedQPS) + } + if got := c.Stats(base.Add(5*time.Second), base.Add(10*time.Second)); got.AchievedQPS != 1 { + t.Errorf("retained interval reports %v achieved, want 1/s", got.AchievedQPS) + } +} + +func TestPacerHoldsItsSchedule(t *testing.T) { + // The pacer must emit the requested count at the requested spacing, and + // must do it without letting a slow server slow the schedule down. + sched := &Schedule{} + c := NewCollector(sched) + p := &Pacer{Collector: c, MaxInFlight: 10_000, TickCap: time.Millisecond} + + const rate = 500.0 + const hold = 400 * time.Millisecond + rung := sched.Begin(Rung{RateQPS: rate, Hold: hold}, time.Now()) + + var sent atomic.Int64 + start := time.Now() + if err := p.RunRung(context.Background(), rung, func(ctx context.Context) (Outcome, int) { + sent.Add(1) + // Every response takes far longer than the inter-arrival gap. A closed + // loop would collapse to ~1 request per 50ms here. + time.Sleep(50 * time.Millisecond) + return OutcomeOK, 200 + }); err != nil { + t.Fatalf("RunRung: %v", err) + } + elapsed := time.Since(start) + + // The dispatch loop should finish when the schedule does, not when the + // responses do. + if elapsed > hold+150*time.Millisecond { + t.Errorf("RunRung took %v for a %v rung: the loop is waiting on responses", elapsed, hold) + } + if !p.Drain(context.Background(), 5*time.Second) { + t.Fatalf("in-flight did not drain: %d left", c.InFlight()) + } + + want := int64(rate * hold.Seconds()) + if got := sent.Load(); got != want { + t.Errorf("dispatched %d requests, want %d", got, want) + } + + stats := c.Stats(rung.StartAt, rung.End()) + if math.Abs(stats.OfferedQPS-rate) > 1 { + t.Errorf("offered = %v, want %v", stats.OfferedQPS, rate) + } + // Real timers, so allow slack; the point is that lag is small and measured, + // not that it is zero. + if stats.DispatchLag.P95Ms > 25 { + t.Errorf("dispatch lag p95 = %vms; the pacer is not keeping its own schedule", stats.DispatchLag.P95Ms) + } +} + +func TestPacerShedsRatherThanBlockingAtTheInFlightCap(t *testing.T) { + // Blocking would turn the open loop closed at the worst possible moment. + // Shedding keeps the schedule and records that the generator, not the + // system, dropped the request. + sched := &Schedule{} + c := NewCollector(sched) + p := &Pacer{Collector: c, MaxInFlight: 5, TickCap: time.Millisecond} + + release := make(chan struct{}) + var wg sync.WaitGroup + rung := sched.Begin(Rung{RateQPS: 500, Hold: 200 * time.Millisecond}, time.Now()) + + wg.Add(1) + go func() { + defer wg.Done() + _ = p.RunRung(context.Background(), rung, func(ctx context.Context) (Outcome, int) { + <-release + return OutcomeOK, 200 + }) + }() + wg.Wait() + close(release) + + if !p.Drain(context.Background(), 5*time.Second) { + t.Fatalf("did not drain: %d in flight", c.InFlight()) + } + stats := c.Stats(rung.StartAt, rung.End().Add(time.Second)) + if stats.Outcomes[OutcomeShed] == 0 { + t.Fatalf("nothing was shed despite a cap of 5 and 100 requests: %+v", stats.Outcomes) + } + if got := c.InFlight(); got != 0 { + t.Errorf("in-flight = %d after drain, want 0 — shed requests must not leak the counter", got) + } +} + +func TestPacerStopsOnContextCancel(t *testing.T) { + sched := &Schedule{} + c := NewCollector(sched) + p := &Pacer{Collector: c, MaxInFlight: 1000, TickCap: time.Millisecond} + rung := sched.Begin(Rung{RateQPS: 100, Hold: time.Minute}, time.Now()) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + err := p.RunRung(ctx, rung, func(ctx context.Context) (Outcome, int) { return OutcomeOK, 200 }) + if err == nil { + t.Error("RunRung returned nil on a cancelled context, want the context error") + } + if time.Since(start) > 2*time.Second { + t.Errorf("RunRung ran %v past cancellation", time.Since(start)) + } +} diff --git a/internal/benchmarking/routercap/promtext.go b/internal/benchmarking/routercap/promtext.go new file mode 100644 index 000000000..a5130bfad --- /dev/null +++ b/internal/benchmarking/routercap/promtext.go @@ -0,0 +1,196 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A small scanner for the Prometheus text format. cAdvisor, Envoy and the sidecar +// all speak it, and none of them needs the full client library to be read. + +package routercap + +import ( + "bufio" + "fmt" + "io" + "math" + "strconv" + "strings" +) + +// promSample is one line of Prometheus text exposition. +type promSample struct { + Name string + Labels map[string]string + Value float64 + // TimestampMs is the sample's own timestamp, or 0 when the exposition + // omits one. The kubelet's cAdvisor endpoint sets it, and the harness + // depends on it to know when a measurement was actually taken. + TimestampMs int64 +} + +// scanPromText streams r and invokes fn for each sample whose metric name is in +// want. +func scanPromText(r io.Reader, want map[string]bool, fn func(promSample)) error { + return scanPromTextMatch(r, func(name string) bool { return want[name] }, fn) +} + +// scanPromTextMatch streams r and invokes fn for each sample whose metric name +// satisfies match. It decides on the name before parsing anything else because +// a busy node's cAdvisor payload runs to megabytes every few seconds. +func scanPromTextMatch(r io.Reader, match func(name string) bool, fn func(promSample)) error { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + line := 0 + for sc.Scan() { + line++ + text := sc.Text() + if text == "" || text[0] == '#' { + continue + } + // Cheap name prefix check before any allocation. + end := strings.IndexAny(text, "{ ") + if end < 0 { + continue + } + name := text[:end] + if !match(name) { + continue + } + s, err := parsePromLine(name, text[end:]) + if err != nil { + return fmt.Errorf("line %d (%s): %w", line, name, err) + } + fn(s) + } + return sc.Err() +} + +// parsePromLine parses the remainder of an exposition line after the metric +// name: an optional {label set}, a value, and an optional timestamp. +func parsePromLine(name, rest string) (promSample, error) { + s := promSample{Name: name, Labels: map[string]string{}} + if strings.HasPrefix(rest, "{") { + close := findLabelSetEnd(rest) + if close < 0 { + return s, fmt.Errorf("unterminated label set") + } + if err := parseLabels(rest[1:close], s.Labels); err != nil { + return s, err + } + rest = rest[close+1:] + } + fields := strings.Fields(rest) + if len(fields) == 0 { + return s, fmt.Errorf("no value") + } + v, err := parsePromValue(fields[0]) + if err != nil { + return s, fmt.Errorf("value %q: %w", fields[0], err) + } + s.Value = v + if len(fields) > 1 { + ts, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return s, fmt.Errorf("timestamp %q: %w", fields[1], err) + } + s.TimestampMs = ts + } + return s, nil +} + +// findLabelSetEnd returns the index of the '}' closing the label set, skipping +// braces that appear inside quoted label values. +func findLabelSetEnd(s string) int { + inQuote, escaped := false, false + for i := 1; i < len(s); i++ { + switch { + case escaped: + escaped = false + case s[i] == '\\' && inQuote: + escaped = true + case s[i] == '"': + inQuote = !inQuote + case s[i] == '}' && !inQuote: + return i + } + } + return -1 +} + +// parseLabels splits a comma-separated label list into out. +func parseLabels(s string, out map[string]string) error { + for len(s) > 0 { + s = strings.TrimLeft(s, " ,") + if s == "" { + return nil + } + eq := strings.IndexByte(s, '=') + if eq < 0 { + return fmt.Errorf("label %q has no '='", s) + } + key := strings.TrimSpace(s[:eq]) + s = s[eq+1:] + if len(s) == 0 || s[0] != '"' { + return fmt.Errorf("label %q value is not quoted", key) + } + val, n, err := unquoteLabelValue(s) + if err != nil { + return fmt.Errorf("label %q: %w", key, err) + } + out[key] = val + s = s[n:] + } + return nil +} + +// unquoteLabelValue decodes a quoted label value starting at s[0] == '"', +// returning the value and how many bytes it consumed including both quotes. +func unquoteLabelValue(s string) (string, int, error) { + var b strings.Builder + for i := 1; i < len(s); i++ { + switch s[i] { + case '\\': + if i+1 >= len(s) { + return "", 0, fmt.Errorf("trailing escape") + } + i++ + switch s[i] { + case 'n': + b.WriteByte('\n') + case '\\', '"': + b.WriteByte(s[i]) + default: + b.WriteByte('\\') + b.WriteByte(s[i]) + } + case '"': + return b.String(), i + 1, nil + default: + b.WriteByte(s[i]) + } + } + return "", 0, fmt.Errorf("unterminated value") +} + +// parsePromValue handles the three spellings Prometheus allows for +// non-finite values alongside ordinary floats. +func parsePromValue(s string) (float64, error) { + switch s { + case "+Inf": + return math.Inf(1), nil + case "-Inf": + return math.Inf(-1), nil + case "NaN": + return math.NaN(), nil + } + return strconv.ParseFloat(s, 64) +} diff --git a/internal/benchmarking/routercap/record.go b/internal/benchmarking/routercap/record.go new file mode 100644 index 000000000..d491bab31 --- /dev/null +++ b/internal/benchmarking/routercap/record.go @@ -0,0 +1,234 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The Sample record — one window of load, latency, CPU and memory, aligned — and the +// ephemeral-port budget arithmetic that goes into it. + +package routercap + +import ( + "sort" + "time" +) + +// Roles name the containers the run watches, so the output is keyed by what a +// container is rather than by a pod name that changes on every arm rollout. +const ( + RoleEnvoy = "envoy" + RoleSidecar = "atenet-router" + RoleLoadgen = "loadgen" + RoleControlPlane = "ate-system" + RoleWorker = "worker" +) + +// Target is one container the sampler watches and the role it plays. +type Target struct { + Role string + Key ContainerKey +} + +// GroupUsage aggregates a role that has many containers — the control plane and +// the worker pods. Sums for the resources, maxima for throttling: averaging a +// single throttled container across a dozen idle ones would hide it. +type GroupUsage struct { + Containers int `json:"containers"` + CPUCores float64 `json:"cpu_cores"` + // CPUUtilizationMax is the worst single container's use against its own + // limit. The sum would be meaningless across differently-sized containers. + CPUUtilizationMax float64 `json:"cpu_utilization_max"` + CPUUtilizationMaxOf string `json:"cpu_utilization_max_of,omitempty"` + MemoryWorkingSetBytes float64 `json:"memory_working_set_bytes"` + ThrottledPeriods float64 `json:"throttled_periods"` + ThrottledSeconds float64 `json:"throttled_seconds"` + ThrottledFractionMax float64 `json:"throttled_fraction_max"` + ThrottledMaxOf string `json:"throttled_max_of,omitempty"` +} + +func aggregate(us []ContainerUsage) GroupUsage { + g := GroupUsage{Containers: len(us)} + for _, u := range us { + g.CPUCores += u.CPUCores + g.MemoryWorkingSetBytes += u.MemoryWorkingSetBytes + g.ThrottledPeriods += u.ThrottledPeriods + g.ThrottledSeconds += u.ThrottledSeconds + if u.CPUUtilization > g.CPUUtilizationMax { + g.CPUUtilizationMax, g.CPUUtilizationMaxOf = u.CPUUtilization, u.Container + } + if u.ThrottledFraction > g.ThrottledFractionMax { + g.ThrottledFractionMax, g.ThrottledMaxOf = u.ThrottledFraction, u.Container + } + } + return g +} + +// PortBudget is the ephemeral-port picture for the router pod. The range is +// read out of the live pod at setup, not assumed to be the Linux default. +type PortBudget struct { + RangeLow int `json:"range_low"` + RangeHigh int `json:"range_high"` + // Available is the size of the range: the hard ceiling on simultaneously + // held source ports for a given destination-less estimate. + Available int `json:"available"` + + // ActiveConnections is Envoy's upstream_cx_active on the actor cluster. + // The upstream hop is HTTP/1.1, so this is also the count of source ports + // currently bound. + ActiveConnections float64 `json:"active_connections"` + // TimeWaitEstimate is new connections over the last minute, the TIME_WAIT + // linger for a closed port. Estimated from the window's connection rate + // because nothing in the pod exports a TIME_WAIT count. + TimeWaitEstimate float64 `json:"time_wait_estimate"` + EstimatedInUse float64 `json:"estimated_in_use"` + Headroom float64 `json:"headroom"` + Utilization float64 `json:"utilization"` + + // CircuitBreakerLimit is the configured concurrency cap, deliberately below + // Available so Envoy's counted overflow trips before the kernel's opaque + // EADDRNOTAVAIL. Carrying both lets a reader confirm that ordering held. + CircuitBreakerLimit int `json:"circuit_breaker_limit"` +} + +// timeWaitSeconds is the kernel's fixed 2*MSL linger for a closed connection. +// Not tunable without a kernel rebuild, so it is a constant rather than a knob. +const timeWaitSeconds = 60 + +func portBudget(rangeLow, rangeHigh, breakerLimit int, cxActive, newConnsPerSec float64) PortBudget { + p := PortBudget{ + RangeLow: rangeLow, + RangeHigh: rangeHigh, + ActiveConnections: cxActive, + TimeWaitEstimate: newConnsPerSec * timeWaitSeconds, + CircuitBreakerLimit: breakerLimit, + } + if rangeHigh >= rangeLow && rangeLow > 0 { + p.Available = rangeHigh - rangeLow + 1 + } + p.EstimatedInUse = p.ActiveConnections + p.TimeWaitEstimate + if p.Available > 0 { + p.Headroom = float64(p.Available) - p.EstimatedInUse + p.Utilization = p.EstimatedInUse / float64(p.Available) + } + return p +} + +// Sample is one line of samples.jsonl: everything true of one interval, from +// every source, over the same [T0, T1). AlignmentSpreadMs is the largest +// disagreement between any contributing container's own cAdvisor interval and +// that pair, so the alignment claim can be checked against the data. +type Sample struct { + Arm int `json:"arm_cores"` + Pass int `json:"pass"` + Rung int `json:"rung"` + // RungQPS is the rung's nominal rate. Load.OfferedQPS is what the schedule + // actually asked for over this window, which differs at a rung boundary. + RungQPS float64 `json:"rung_qps"` + // Warmup marks a window inside a rung's discarded head, kept in the file + // so a chart can show the settling and an analysis can exclude it. + Warmup bool `json:"warmup"` + + // T is the interval midpoint: the x value when a chart must draw an + // interval as a point. T0 and T1 are carried so it need not. + T time.Time `json:"t"` + T0 time.Time `json:"t0"` + T1 time.Time `json:"t1"` + WindowSeconds float64 `json:"window_seconds"` + // WindowPolls is how many cAdvisor fetches the window took. One means the + // kubelet moved faster than the poll interval, so the resolution here is + // poll-limited rather than kubelet-limited. + WindowPolls int `json:"window_polls"` + AlignmentSpreadMs float64 `json:"alignment_spread_ms"` + + Load GenStats `json:"load"` + // Client is the generator measuring its own transport. A generator + // churning connections is heading for its own port wall, and that cliff + // would be the rig's rather than the router's. + Client ClientStats `json:"client"` + + // Containers holds the single-container roles, keyed by role. + Containers map[string]ContainerUsage `json:"containers"` + // Groups holds the many-container roles, keyed by role. + Groups map[string]GroupUsage `json:"groups,omitempty"` + + Envoy *EnvoyDelta `json:"envoy,omitempty"` + Router *RouterDelta `json:"router,omitempty"` + Ports PortBudget `json:"ports"` + // Spans divides the mean request across the hops of the request path. Nil + // when Envoy reported no request-time samples for the window. + Spans *LatencySpans `json:"spans,omitempty"` + + Guards []GuardTrip `json:"guards,omitempty"` + // Missing names containers cAdvisor did not report this window. Present so + // "the router used no CPU" and "we could not see the router" never look the + // same in the output. + Missing []string `json:"missing,omitempty"` + // Errors are non-fatal problems encountered building this record. + Errors []string `json:"errors,omitempty"` +} + +// FineSample is one line of fine.jsonl: the generator's own series at 1s, +// which resolves cliffs faster than the kubelet's ~10s housekeeping. It +// deliberately carries no resource fields, so it cannot be plotted against a +// resource panel and imply an alignment that does not exist. +type FineSample struct { + Arm int `json:"arm_cores"` + Pass int `json:"pass"` + Rung int `json:"rung"` + RungQPS float64 `json:"rung_qps"` + Warmup bool `json:"warmup"` + T time.Time `json:"t"` + T0 time.Time `json:"t0"` + T1 time.Time `json:"t1"` + Load GenStats `json:"load"` +} + +// buildSample assembles a record from a window and the sources sampled around +// it. Container usage is split into single roles and aggregated groups, and +// anything missing or unreadable is recorded on the sample rather than dropped. +func buildSample(w Window, targets []Target) (containers map[string]ContainerUsage, groups map[string]GroupUsage, spread time.Duration, missing []string, errs []string) { + keys := make([]ContainerKey, 0, len(targets)) + byKey := make(map[ContainerKey]string, len(targets)) + for _, t := range targets { + keys = append(keys, t.Key) + byKey[t.Key] = t.Role + } + usage, spread, missingKeys, uerrs := w.Usage(keys) + + for _, k := range missingKeys { + missing = append(missing, byKey[k]+"="+k.String()) + } + sort.Strings(missing) + for _, e := range uerrs { + errs = append(errs, e.Error()) + } + sort.Strings(errs) + + containers = map[string]ContainerUsage{} + grouped := map[string][]ContainerUsage{} + for k, u := range usage { + role := byKey[k] + switch role { + case RoleControlPlane, RoleWorker: + grouped[role] = append(grouped[role], u) + default: + containers[role] = u + } + } + if len(grouped) > 0 { + groups = map[string]GroupUsage{} + for role, us := range grouped { + groups[role] = aggregate(us) + } + } + return containers, groups, spread, missing, errs +} diff --git a/internal/benchmarking/routercap/record_test.go b/internal/benchmarking/routercap/record_test.go new file mode 100644 index 000000000..c36843bcf --- /dev/null +++ b/internal/benchmarking/routercap/record_test.go @@ -0,0 +1,233 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for sample assembly, the port budget, and the JSON shape charts.py reads. + +package routercap + +import ( + "encoding/json" + "math" + "strings" + "testing" + "time" +) + +func TestAggregateKeepsMaximaForThrottling(t *testing.T) { + // One throttled container among many idle ones must survive aggregation. + // Averaging is what would hide it. + g := aggregate([]ContainerUsage{ + {Container: "ate-controller", CPUCores: 0.2, CPUUtilization: 0.10}, + {Container: "ate-api-server", CPUCores: 15, CPUUtilization: 0.94, + ThrottledPeriods: 40, Periods: 100, ThrottledSeconds: 0.2, ThrottledFraction: 0.4}, + {Container: "coredns", CPUCores: 0.1, CPUUtilization: 0.10}, + }) + + if g.Containers != 3 { + t.Errorf("Containers = %d, want 3", g.Containers) + } + if want := 15.3; math.Abs(g.CPUCores-want) > 1e-9 { + t.Errorf("CPUCores = %v, want %v (sum)", g.CPUCores, want) + } + if g.ThrottledFractionMax != 0.4 || g.ThrottledMaxOf != "ate-api-server" { + t.Errorf("throttling max = %v of %q, want 0.4 of ate-api-server", g.ThrottledFractionMax, g.ThrottledMaxOf) + } + if g.CPUUtilizationMax != 0.94 || g.CPUUtilizationMaxOf != "ate-api-server" { + t.Errorf("utilization max = %v of %q, want 0.94 of ate-api-server", g.CPUUtilizationMax, g.CPUUtilizationMaxOf) + } +} + +func TestPortBudget(t *testing.T) { + // The Linux default range, the configured breaker, 5000 live upstream + // connections and 100 new per second. + p := portBudget(32768, 60999, 20000, 5000, 100) + + if p.Available != 28232 { + t.Errorf("Available = %d, want 28232", p.Available) + } + if p.TimeWaitEstimate != 6000 { + t.Errorf("TimeWaitEstimate = %v, want 6000 (100/s over a 60s TIME_WAIT)", p.TimeWaitEstimate) + } + if p.EstimatedInUse != 11000 { + t.Errorf("EstimatedInUse = %v, want 11000", p.EstimatedInUse) + } + if p.Headroom != 17232 { + t.Errorf("Headroom = %v, want 17232", p.Headroom) + } + // The ordering the whole design depends on: Envoy's counted cap trips + // before the kernel's opaque one. + if p.CircuitBreakerLimit >= p.Available { + t.Errorf("circuit breaker %d is not below the port budget %d; the kernel would run out first and the failure would be an unattributable EADDRNOTAVAIL", + p.CircuitBreakerLimit, p.Available) + } +} + +func TestPortBudgetWithAnUnreadRange(t *testing.T) { + // If the range could not be read out of the live pod, no utilization is + // better than one computed against an assumed default. + p := portBudget(0, 0, 20000, 5000, 100) + if p.Available != 0 || p.Utilization != 0 || p.Headroom != 0 { + t.Errorf("an unread port range produced derived values: %+v", p) + } + if p.EstimatedInUse != 11000 { + t.Errorf("EstimatedInUse = %v, want the raw estimate to survive", p.EstimatedInUse) + } +} + +func TestBuildSampleSplitsRolesAndNamesWhatIsMissing(t *testing.T) { + t0 := time.UnixMilli(1700000000000) + t1 := t0.Add(10 * time.Second) + + mk := func(k ContainerKey, at time.Time, cpu float64) ContainerSample { + return ContainerSample{Key: k, At: at, CPUSecondsTotal: cpu, CPUQuota: 4000000, CPUPeriod: 100000} + } + envoy := ContainerKey{"ate-system", "atenet-router-abc", "envoy"} + sidecar := ContainerKey{"ate-system", "atenet-router-abc", "atenet-router"} + api := ContainerKey{"ate-system", "ate-api-server-1", "ate-api-server"} + ctrl := ContainerKey{"ate-system", "ate-controller-1", "ate-controller"} + gone := ContainerKey{"benchmarking", "routercap-xyz", "loadgen"} + + w := Window{ + T0: t0, T1: t1, + Prev: CadvisorScrape{Containers: map[ContainerKey]ContainerSample{ + envoy: mk(envoy, t0, 100), + sidecar: mk(sidecar, t0, 50), + api: mk(api, t0, 10), + ctrl: mk(ctrl, t0, 1), + }}, + Cur: CadvisorScrape{Containers: map[ContainerKey]ContainerSample{ + envoy: mk(envoy, t1, 120), + sidecar: mk(sidecar, t1, 60), + api: mk(api, t1, 15), + ctrl: mk(ctrl, t1, 1.1), + }}, + } + targets := []Target{ + {RoleEnvoy, envoy}, {RoleSidecar, sidecar}, + {RoleControlPlane, api}, {RoleControlPlane, ctrl}, + {RoleLoadgen, gone}, + } + + containers, groups, spread, missing, errs := buildSample(w, targets) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if got := containers[RoleEnvoy].CPUCores; got != 2 { + t.Errorf("envoy CPUCores = %v, want 2 (20 cpu-seconds over 10s)", got) + } + if _, ok := containers[RoleControlPlane]; ok { + t.Error("a many-container role leaked into Containers instead of Groups") + } + if g := groups[RoleControlPlane]; g.Containers != 2 || math.Abs(g.CPUCores-0.51) > 1e-9 { + t.Errorf("control plane group = %+v, want 2 containers summing to 0.51 cores", g) + } + // The loadgen container was never in either scrape. It must be named, not + // dropped, or "we could not see it" reads as "it used nothing". + if len(missing) != 1 || !strings.Contains(missing[0], RoleLoadgen) { + t.Errorf("missing = %v, want the loadgen container named", missing) + } + if _, ok := containers[RoleLoadgen]; ok { + t.Error("a missing container produced a usage entry") + } + if spread != 0 { + t.Errorf("spread = %v, want 0: every container shared the anchor's timestamps", spread) + } +} + +func TestBuildSampleReportsSpreadAgainstTheAnchor(t *testing.T) { + // A container whose own cAdvisor timestamps differ from the anchor's is + // still measured, but the disagreement has to reach the output so the + // alignment claim can be checked rather than assumed. + t0 := time.UnixMilli(1700000000000) + t1 := t0.Add(10 * time.Second) + envoy := ContainerKey{"ate-system", "atenet-router-abc", "envoy"} + worker := ContainerKey{"benchmark-workloads", "glutton-7", "ateom"} + + w := Window{ + T0: t0, T1: t1, + Prev: CadvisorScrape{Containers: map[ContainerKey]ContainerSample{ + envoy: {Key: envoy, At: t0, CPUSecondsTotal: 100}, + worker: {Key: worker, At: t0.Add(-1500 * time.Millisecond), CPUSecondsTotal: 5}, + }}, + Cur: CadvisorScrape{Containers: map[ContainerKey]ContainerSample{ + envoy: {Key: envoy, At: t1, CPUSecondsTotal: 120}, + worker: {Key: worker, At: t1.Add(-1500 * time.Millisecond), CPUSecondsTotal: 6}, + }}, + } + + _, groups, spread, _, errs := buildSample(w, []Target{{RoleEnvoy, envoy}, {RoleWorker, worker}}) + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if spread != 1500*time.Millisecond { + t.Errorf("spread = %v, want 1.5s", spread) + } + if groups[RoleWorker].Containers != 1 { + t.Errorf("worker group = %+v, want 1 container", groups[RoleWorker]) + } +} + +func TestSampleRoundTripsThroughJSON(t *testing.T) { + // samples.jsonl is the deliverable and charts.py reads nothing else, so the + // interval bounds and every headline series must survive the encoding. + t0 := time.UnixMilli(1700000000000).UTC() + t1 := t0.Add(10 * time.Second) + in := Sample{ + Arm: 40, Rung: 3, RungQPS: 4000, + T: t0.Add(5 * time.Second), T0: t0, T1: t1, WindowSeconds: 10, WindowPolls: 4, + Load: GenStats{ + OfferedQPS: 4000, AchievedQPS: 3990, InFlightEnd: 52, + Latency: LatencyStats{Count: 39900, P50Ms: 4.1, P95Ms: 21.3}, + Outcomes: map[Outcome]int{OutcomeOK: 39900}, + }, + Containers: map[string]ContainerUsage{ + RoleEnvoy: {Container: "envoy", CPUCores: 22.5, CPULimitCores: 40, CPUUtilization: 0.5625}, + RoleSidecar: {Container: "atenet-router", CPUCores: 4.1, CPULimitCores: 8}, + }, + Ports: portBudget(32768, 60999, 20000, 52, 3), + } + + b, err := json.Marshal(in) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var out Sample + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if !out.T0.Equal(t0) || !out.T1.Equal(t1) { + t.Errorf("interval bounds = %s..%s, want %s..%s", out.T0, out.T1, t0, t1) + } + if out.Load.OfferedQPS != 4000 || out.Load.Latency.P95Ms != 21.3 { + t.Errorf("load did not survive: %+v", out.Load) + } + if out.Containers[RoleEnvoy].CPUCores != 22.5 { + t.Errorf("envoy cpu did not survive: %+v", out.Containers) + } + if out.Ports.Available != 28232 { + t.Errorf("port budget did not survive: %+v", out.Ports) + } + // Guards and the two scraper sections are absent here and must not appear + // as nulls a chart would have to special-case. Checked on the encoded form + // as well as the decoded one, since a null decodes back to nil either way. + if out.Envoy != nil || out.Router != nil || out.Guards != nil { + t.Errorf("absent optional sections decoded as non-nil: envoy=%v router=%v guards=%v", out.Envoy, out.Router, out.Guards) + } + for _, key := range []string{`"guards"`, `"router":`, `,"envoy":{"concurrency"`} { + if strings.Contains(string(b), key) { + t.Errorf("empty optional section %s was encoded: %s", key, b) + } + } +} diff --git a/internal/benchmarking/routercap/run.go b/internal/benchmarking/routercap/run.go new file mode 100644 index 000000000..13cfbeb1a --- /dev/null +++ b/internal/benchmarking/routercap/run.go @@ -0,0 +1,477 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The runner: primes the actors, walks the ladder, and emits one aligned sample per +// cAdvisor window until the schedule ends or a fatal guard stops it. + +package routercap + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" +) + +// Client is what the runner needs from the load generator: a way to issue one +// request, and a way for the generator to report on itself. *Sender is the +// implementation; the interface exists so the orchestration can be tested +// without a socket. +type Client interface { + Send(ctx context.Context) (Outcome, int) + Stats() ClientStats +} + +// Runner executes one arm: one Envoy CPU size, one ladder, one pair of output +// streams. Arms are separate processes, so this binary needs no write access +// to anything in the cluster. +type Runner struct { + // Arm is the Envoy container's CPU limit in cores, and Pass distinguishes + // repeats of the same arm. Both are stamped on every record so four arms' + // files concatenate into one dataset without losing which is which. + Arm int + Pass int + + Rungs []Rung + Client Client + Sink Sink + + Windows *WindowDriver + // Envoy, Contention and Router are optional. A nil one leaves its section off every + // record rather than filling it with zeros. + Envoy *EnvoyClient + Contention *ContentionClient + Router *RouterClient + + Targets []Target + Guards GuardConfig + + PortRange PortRange + CircuitBreakerLimit int + + // MaxInFlight bounds the generator's own concurrency; reaching it is a rig + // failure recorded as shed requests, not a result. + MaxInFlight int64 + // TickCap bounds the pacer's sleep and so bounds the dispatch lag the + // dispatch loop itself can introduce. + TickCap time.Duration + // FineInterval is the cadence of the generator-only series. Zero means 1s. + FineInterval time.Duration + // DrainTimeout bounds the wait for in-flight requests at the end of the + // ladder. Whether it emptied is recorded, because an arm that ended with + // requests outstanding hands them to whatever runs next. + DrainTimeout time.Duration + + Log *slog.Logger + + sched *Schedule + collector *Collector + + // Envoy and router scrapes are taken and differenced entirely inside the + // sampler goroutine, so they need no lock. + prevEnvoy EnvoyStats + prevRouter RouterStats + prevContention ContentionStats + haveEnvoy bool + haveRouter bool + haveContention bool + + mu sync.Mutex + fineFrontier time.Time + windowFrontier time.Time +} + +// RunResult is what one arm produced, for the run header. +type RunResult struct { + Arm int `json:"arm_cores"` + Pass int `json:"pass"` + + Rungs []Rung `json:"rungs"` + Windows int `json:"windows"` + FineSamples int `json:"fine_samples"` + + // EnvoyConcurrency is the worker-thread count Envoy reported; it must + // equal Arm. Left unset, Envoy sizes it from the node's core count, and + // the arm measures CFS throttling instead of the proxy. + EnvoyConcurrency float64 `json:"envoy_concurrency"` + // ClockSkewMs is the residual error in the alignment claim; see + // WindowDriver.Skew. + ClockSkewMs float64 `json:"clock_skew_ms"` + + // Drained says whether every request had completed when the arm ended. + Drained bool `json:"drained"` + // Interrupted marks an arm cut short by its context rather than by the + // ladder finishing. + Interrupted bool `json:"interrupted,omitempty"` + FatalTrips []GuardTrip `json:"fatal_trips,omitempty"` +} + +func (r *Runner) log() *slog.Logger { + if r.Log != nil { + return r.Log + } + return slog.Default() +} + +func (r *Runner) validate() error { + switch { + case r.Client == nil: + return fmt.Errorf("runner needs a client") + case r.Sink == nil: + return fmt.Errorf("runner needs a sink") + case r.Windows == nil: + return fmt.Errorf("runner needs a window driver: the whole series is aligned off its clock") + case len(r.Rungs) == 0: + return fmt.Errorf("runner needs at least one rung") + } + return nil +} + +// Run executes the arm and returns once the ladder has finished, the guards +// have stopped it, or ctx is cancelled. Three concurrent parts — the pacer, +// the cAdvisor-clocked sampler, and the 1s generator-only series — share one +// collector of raw request events. +func (r *Runner) Run(ctx context.Context) (RunResult, error) { + res := RunResult{Arm: r.Arm, Pass: r.Pass} + if err := r.validate(); err != nil { + return res, err + } + + r.sched = &Schedule{} + r.collector = NewCollector(r.sched) + + if err := r.prime(ctx); err != nil { + return res, err + } + if skew, ok := r.Windows.Skew(); ok { + res.ClockSkewMs = float64(skew) / float64(time.Millisecond) + } + + // loadCtx is cancelled by a fatal guard; the sampler keeps its own ctx so + // it can still write the record that explains why the load stopped. + loadCtx, stopLoad := context.WithCancel(ctx) + defer stopLoad() + fineCtx, stopFine := context.WithCancel(ctx) + defer stopFine() + + finishing := make(chan struct{}) + // Two wait groups, not one: the fine loop stops only after the sampler has + // taken its last window, so waiting on both together would deadlock. + var ( + samplerWG, fineWG sync.WaitGroup + sampleErr error + ) + + samplerWG.Add(1) + go func() { + defer samplerWG.Done() + sampleErr = r.sampleLoop(ctx, finishing, stopLoad, &res) + }() + fineWG.Add(1) + go func() { + defer fineWG.Done() + r.fineLoop(fineCtx, &res) + }() + + pacer := &Pacer{Collector: r.collector, MaxInFlight: r.MaxInFlight, TickCap: r.TickCap} + ladderErr := r.runLadder(loadCtx, pacer) + res.Rungs = r.sched.Rungs() + + drainTimeout := r.DrainTimeout + if drainTimeout <= 0 { + drainTimeout = 30 * time.Second + } + res.Drained = pacer.Drain(ctx, drainTimeout) + if !res.Drained { + r.log().Warn("arm ended with requests still in flight", + "arm", r.Arm, "pass", r.Pass, "in_flight", r.collector.InFlight()) + } + + // One more window after the load stops, so the ladder's final rung is + // covered by an aligned record rather than truncated mid-interval. The fine + // series keeps running through that wait and stops only afterwards. + close(finishing) + samplerWG.Wait() + stopFine() + fineWG.Wait() + + switch { + case sampleErr != nil: + return res, sampleErr + case ladderErr != nil && !errors.Is(ladderErr, context.Canceled): + return res, ladderErr + case ctx.Err() != nil: + res.Interrupted = true + return res, ctx.Err() + } + return res, nil +} + +// prime establishes the first boundary for every differenced source, so the +// first emitted window is a real interval rather than a delta against zero. +func (r *Runner) prime(ctx context.Context) error { + if err := r.Windows.Prime(ctx); err != nil { + return fmt.Errorf("prime cadvisor window: %w", err) + } + if r.Envoy != nil { + s, err := r.Envoy.Scrape(ctx) + if err != nil { + // Fatal here, unlike mid-run: an admin endpoint unreachable before + // any load has been offered is a broken rig. + return fmt.Errorf("prime envoy stats: %w", err) + } + r.prevEnvoy, r.haveEnvoy = s, true + } + if r.Router != nil { + s, err := r.Router.Scrape(ctx) + if err != nil { + return fmt.Errorf("prime router stats: %w", err) + } + r.prevRouter, r.haveRouter = s, true + } + return nil +} + +// runLadder walks the rungs back to back. Rungs are not drained between steps: +// idling the system at a boundary would make the next rung's first seconds +// measure a cold connection pool rather than a running one. +func (r *Runner) runLadder(ctx context.Context, p *Pacer) error { + for _, rung := range r.Rungs { + if err := ctx.Err(); err != nil { + return err + } + started := r.sched.Begin(rung, time.Now()) + r.log().Info("rung start", + "arm", r.Arm, "pass", r.Pass, "rung", started.Index, + "offered_qps", started.RateQPS, "hold", started.Hold) + if err := p.RunRung(ctx, started, r.Client.Send); err != nil { + return err + } + } + return nil +} + +// sampleLoop emits the aligned series. It ticks off cAdvisor's housekeeping +// clock rather than a local timer, which is the whole reason a vertical line +// through the four panels describes one moment. +func (r *Runner) sampleLoop(ctx context.Context, finishing <-chan struct{}, stopLoad func(), res *RunResult) error { + for { + w, err := r.Windows.Next(ctx) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return nil + } + return err + } + + s := r.buildRecord(ctx, w) + res.Windows++ + if s.Envoy != nil && s.Envoy.Concurrency > 0 { + res.EnvoyConcurrency = s.Envoy.Concurrency + } + if err := r.Sink.Sample(s); err != nil { + return fmt.Errorf("write sample: %w", err) + } + r.noteFrontier(false, w.T1) + r.prune() + + if fatal := FatalTrips(s.Guards); len(fatal) > 0 { + // Stop the load first, then report. The record that names the trip + // is already written, so the run directory explains itself. + res.FatalTrips = fatal + stopLoad() + for _, t := range fatal { + r.log().Error("rig guard tripped", "arm", r.Arm, "guard", t.Guard, "detail", t.Detail) + } + return &RigLimitedError{Trips: fatal} + } + + select { + case <-finishing: + return nil + default: + } + } +} + +// buildRecord assembles one aligned sample. Every source is asked for the same +// [T0, T1) the window defines, and anything that could not be read lands in +// Missing or Errors rather than being silently zero. +func (r *Runner) buildRecord(ctx context.Context, w Window) Sample { + containers, groups, spread, missing, errs := buildSample(w, r.Targets) + + s := Sample{ + Arm: r.Arm, + Pass: r.Pass, + Rung: -1, + T: w.Mid(), + T0: w.T0, + T1: w.T1, + WindowSeconds: w.Duration().Seconds(), + WindowPolls: w.Polls, + AlignmentSpreadMs: float64(spread) / float64(time.Millisecond), + Load: r.collector.Stats(w.T0, w.T1), + Client: r.Client.Stats(), + Containers: containers, + Groups: groups, + Missing: missing, + Errors: errs, + } + // Rung -1 is a window outside any rung: before the first starts or after + // the last ends. Kept, because an idle window immediately after saturation + // is one of the more informative records in the run. + if rung, warm, ok := r.sched.RungAt(s.T); ok { + s.Rung, s.RungQPS, s.Warmup = rung.Index, rung.RateQPS, warm + } + + if r.Envoy != nil { + cur, err := r.Envoy.Scrape(ctx) + switch { + case err != nil: + s.Errors = append(s.Errors, err.Error()) + case !r.haveEnvoy: + r.prevEnvoy, r.haveEnvoy = cur, true + default: + // Rated over the two scrapes' own interval, not the window's: the + // scrapes bracket the window, and using the window's length would + // bias the per-second connection rate the worker guard reads. + secs := cur.At.Sub(r.prevEnvoy.At).Seconds() + d, derr := envoyDelta(r.prevEnvoy, cur, secs) + if derr != nil { + s.Errors = append(s.Errors, derr.Error()) + } else { + s.Envoy = &d + } + r.prevEnvoy = cur + } + } + + // Attached to the Envoy section only when this window has one, so a failed + // fetch reads as absent rather than as zero contention. + if r.Contention != nil && s.Envoy != nil { + cur, err := r.Contention.Scrape(ctx) + switch { + case err != nil: + s.Errors = append(s.Errors, err.Error()) + case !r.haveContention: + r.prevContention, r.haveContention = cur, true + default: + d := contentionDelta(r.prevContention, cur) + s.Envoy.Contention = &d + r.prevContention = cur + } + } + + if r.Router != nil { + cur, err := r.Router.Scrape(ctx) + switch { + case err != nil: + s.Errors = append(s.Errors, err.Error()) + case !r.haveRouter: + r.prevRouter, r.haveRouter = cur, true + default: + d := routerDelta(r.prevRouter, cur) + s.Router = &d + r.prevRouter = cur + } + } + + var cxActive, newCxPerSec float64 + if s.Envoy != nil { + if actor, ok := s.Envoy.Clusters[ActorClusterName]; ok { + cxActive, newCxPerSec = actor.CxActive, actor.NewConnectionsPerSec + } + } + s.Ports = portBudget(r.PortRange.Low, r.PortRange.High, r.CircuitBreakerLimit, cxActive, newCxPerSec) + + // After both scrapes: the breakdown needs Envoy's totals and the sidecar's + // route duration together, and is nil if either is absent. + s.Spans = latencySpans(s.Load, s.Envoy, s.Router) + + // Last, so the guards see the Envoy section they depend on. + s.Guards = r.Guards.Check(&s) + return s +} + +// fineLoop emits the generator-only series: the cliff's shape is faster than +// the kubelet's housekeeping, so the aligned series cannot resolve it. It +// carries no resource fields so it cannot be mistaken for an aligned series. +func (r *Runner) fineLoop(ctx context.Context, res *RunResult) { + interval := r.FineInterval + if interval <= 0 { + interval = time.Second + } + tick := time.NewTicker(interval) + defer tick.Stop() + + last := time.Now() + r.noteFrontier(true, last) + for { + select { + case <-ctx.Done(): + return + case now := <-tick.C: + fs := FineSample{ + Arm: r.Arm, + Pass: r.Pass, + Rung: -1, + T: last.Add(now.Sub(last) / 2), + T0: last, + T1: now, + Load: r.collector.FineStats(last, now), + } + if rung, warm, ok := r.sched.RungAt(fs.T); ok { + fs.Rung, fs.RungQPS, fs.Warmup = rung.Index, rung.RateQPS, warm + } + if err := r.Sink.Fine(fs); err != nil { + // Non-fatal: the fine series is supplementary, and losing it + // must not cost the aligned series the run. + r.log().Warn("write fine sample", "error", err) + } + res.FineSamples++ + last = now + r.noteFrontier(true, last) + } + } +} + +func (r *Runner) noteFrontier(fine bool, t time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + if fine { + r.fineFrontier = t + return + } + r.windowFrontier = t +} + +// prune drops raw request events both consumers have already summarized. The +// cutoff is the *older* of the two frontiers: the fine series runs ~10x ahead +// of the aligned one, and pruning to it would delete the events the aligned +// window is about to read. +func (r *Runner) prune() { + r.mu.Lock() + fine, window := r.fineFrontier, r.windowFrontier + r.mu.Unlock() + if fine.IsZero() || window.IsZero() { + return + } + cutoff := window + if fine.Before(cutoff) { + cutoff = fine + } + r.collector.Prune(cutoff) +} diff --git a/internal/benchmarking/routercap/run_test.go b/internal/benchmarking/routercap/run_test.go new file mode 100644 index 000000000..43e560234 --- /dev/null +++ b/internal/benchmarking/routercap/run_test.go @@ -0,0 +1,730 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// End-to-end tests for the runner against fake cAdvisor, Envoy and actor endpoints. + +package routercap + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +var loadgenKey = ContainerKey{Namespace: "benchmarking", Pod: "routercap-runner-abc", Container: "loadgen"} + +// loadgenFixture adds the generator's own container to a cAdvisor payload. The +// run's most important guard reads it, so a fake node without it would exercise +// the orchestration with that guard silently absent. +func loadgenFixture(at time.Time, cpuSeconds, quota float64) string { + ms := at.UnixMilli() + var b strings.Builder + row := func(metric string, v float64) { + fmt.Fprintf(&b, "%s{container=\"loadgen\",namespace=\"benchmarking\",pod=\"routercap-runner-abc\"} %g %d\n", metric, v, ms) + } + row(metricCPUUsageSeconds, cpuSeconds) + row(metricMemoryWorkingSet, 3e8) + row(metricCFSPeriods, 1000) + row(metricCFSThrottledPeriods, 0) + row(metricSpecCPUQuota, quota) + row(metricSpecCPUPeriod, 100000) + return b.String() +} + +// fakeNode is a kubelet whose housekeeping timestamp advances on a real +// wall-clock grid, so windows the runner produces are genuine intervals of the +// test's own execution. That lets load and CPU statistics in one record +// describe the same moment — the property under test. +type fakeNode struct { + start time.Time + grid time.Duration + + envoyCores float64 + routerCores float64 + loadgenCores float64 + loadgenQuota float64 +} + +func (f *fakeNode) fetch(context.Context) (io.ReadCloser, error) { + k := time.Since(f.start) / f.grid + at := f.start.Add(k * f.grid) + // Counters are a linear function of the *quantized* instant, so every + // derived rate comes out at exactly the configured core count regardless of + // when the fetch happened to land. + secs := at.Sub(f.start).Seconds() + body := cadvisorFixture(at, 100+f.envoyCores*secs, 20+f.routerCores*secs) + + loadgenFixture(at, f.loadgenCores*secs, f.loadgenQuota) + return io.NopCloser(strings.NewReader(body)), nil +} + +// fakeAdmin serves Envoy admin payloads whose counters climb, so consecutive +// scrapes produce a non-degenerate delta. +type fakeAdmin struct { + mu sync.Mutex + n int +} + +func (f *fakeAdmin) fetch(context.Context) (io.ReadCloser, error) { + f.mu.Lock() + n := f.n + f.n++ + f.mu.Unlock() + return io.NopCloser(strings.NewReader(envoyFixture( + 300+float64(n)*10, // cx_total + 900000+float64(n)*5000, // rq_total + 295, // cx_active + ))), nil +} + +func staticFetch(body string) func(context.Context) (io.ReadCloser, error) { + return func(context.Context) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(body)), nil + } +} + +// fakeClient answers instantly and reports whatever transport picture the test +// wants the guards to see. +type fakeClient struct { + mu sync.Mutex + sent int + connsInUse int64 + newConns float64 + reqsPerCx float64 +} + +func (c *fakeClient) Send(context.Context) (Outcome, int) { + c.mu.Lock() + c.sent++ + c.mu.Unlock() + return OutcomeOK, 200 +} + +func (c *fakeClient) Stats() ClientStats { + c.mu.Lock() + defer c.mu.Unlock() + return ClientStats{ + NewConnections: c.newConns, + RequestsPerConnection: c.reqsPerCx, + ConnectionsInUse: c.connsInUse, + } +} + +func (c *fakeClient) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.sent +} + +type memSink struct { + mu sync.Mutex + samples []Sample + fine []FineSample +} + +func (s *memSink) Sample(v Sample) error { + s.mu.Lock() + defer s.mu.Unlock() + s.samples = append(s.samples, v) + return nil +} + +func (s *memSink) Fine(v FineSample) error { + s.mu.Lock() + defer s.mu.Unlock() + s.fine = append(s.fine, v) + return nil +} + +func (s *memSink) all() ([]Sample, []FineSample) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]Sample(nil), s.samples...), append([]FineSample(nil), s.fine...) +} + +// newTestRunner wires a runner against fakes for every external source. The +// grid is small so a ladder that would take minutes in a cluster takes a +// fraction of a second here, without changing any of the logic under test. +func newTestRunner(t *testing.T, client Client, sink Sink, ladder LadderSpec) (*Runner, *fakeNode) { + t.Helper() + node := &fakeNode{ + start: time.Now(), + grid: 60 * time.Millisecond, + envoyCores: 4, + routerCores: 0.5, + loadgenCores: 2, + loadgenQuota: 100 * 100000, // 100 cores + } + guards := DefaultGuardConfig() + guards.WorkerPods = 100 + // The fakes answer instantly, so the generator cannot fall behind for any + // reason a cluster would produce; the tiny grid makes the scheduler's own + // jitter the only source of lag, and it is not what these tests are about. + guards.DispatchLagP95Ms = 0 + + return &Runner{ + Arm: 40, + Pass: 1, + Rungs: ladder.Build(), + Client: client, + Sink: sink, + Windows: &WindowDriver{ + Client: &CadvisorClient{Fetch: node.fetch}, + Anchor: envoyKey, + PollInterval: 3 * time.Millisecond, + MaxWait: 5 * time.Second, + }, + Envoy: &EnvoyClient{Fetch: (&fakeAdmin{}).fetch}, + Router: &RouterClient{Fetch: staticFetch(routerFixture)}, + Targets: []Target{ + {Role: RoleEnvoy, Key: envoyKey}, + {Role: RoleSidecar, Key: routerKey}, + {Role: RoleLoadgen, Key: loadgenKey}, + }, + Guards: guards, + PortRange: DefaultPortRange(), + CircuitBreakerLimit: 20000, + MaxInFlight: 4096, + TickCap: time.Millisecond, + FineInterval: 20 * time.Millisecond, + DrainTimeout: 2 * time.Second, + }, node +} + +func TestRunnerProducesAnAlignedSeries(t *testing.T) { + client := &fakeClient{connsInUse: 40, newConns: 2, reqsPerCx: 500} + sink := &memSink{} + ladder := LadderSpec{StartQPS: 200, StepQPS: 200, Rungs: 3, Hold: 150 * time.Millisecond, Warmup: 40 * time.Millisecond} + r, _ := newTestRunner(t, client, sink, ladder) + + res, err := r.Run(context.Background()) + if err != nil { + t.Fatalf("Run: %v", err) + } + + samples, fine := sink.all() + if len(samples) < 3 { + t.Fatalf("got %d aligned samples over a 450ms ladder on a 60ms grid, want at least 3", len(samples)) + } + if len(fine) == 0 { + t.Fatal("no fine samples written") + } + if res.Windows != len(samples) { + t.Errorf("result counted %d windows but %d were written", res.Windows, len(samples)) + } + if res.FineSamples != len(fine) { + t.Errorf("result counted %d fine samples but %d were written", res.FineSamples, len(fine)) + } + if !res.Drained { + t.Errorf("arm ended with requests still in flight against instantaneous fakes") + } + if len(res.Rungs) != 3 { + t.Errorf("ran %d rungs, want all 3", len(res.Rungs)) + } + + t.Run("EveryRecordDescribesOneRealInterval", func(t *testing.T) { + for i, s := range samples { + if !s.T1.After(s.T0) { + t.Fatalf("sample %d has a degenerate interval [%v, %v)", i, s.T0, s.T1) + } + if want := s.T0.Add(s.T1.Sub(s.T0) / 2); !s.T.Equal(want) { + t.Errorf("sample %d: t = %v, want the interval midpoint %v", i, s.T, want) + } + if got := s.WindowSeconds; math.Abs(got-0.06) > 1e-6 { + t.Errorf("sample %d: window = %vs, want the 60ms kubelet grid", i, got) + } + if i > 0 && !samples[i-1].T1.Equal(s.T0) { + t.Errorf("sample %d starts at %v but the previous ended at %v: the series has a gap", + i, s.T0, samples[i-1].T1) + } + if len(s.Missing) != 0 { + t.Errorf("sample %d reported missing containers: %v", i, s.Missing) + } + if len(s.Errors) != 0 { + t.Errorf("sample %d reported errors: %v", i, s.Errors) + } + } + }) + + t.Run("ResourceSeriesAreReadPerContainer", func(t *testing.T) { + s := samples[len(samples)-1] + if got := s.Containers[RoleEnvoy].CPUCores; math.Abs(got-4) > 1e-6 { + t.Errorf("envoy cpu = %v cores, want 4", got) + } + if got := s.Containers[RoleSidecar].CPUCores; math.Abs(got-0.5) > 1e-6 { + t.Errorf("sidecar cpu = %v cores, want 0.5", got) + } + if got := s.Containers[RoleLoadgen].CPUCores; math.Abs(got-2) > 1e-6 { + t.Errorf("loadgen cpu = %v cores, want 2", got) + } + if got := s.Containers[RoleEnvoy].MemoryWorkingSetBytes; got != 1.5e9 { + t.Errorf("envoy memory = %v, want 1.5e9", got) + } + }) + + t.Run("LoadAndResourcesShareTheWindow", func(t *testing.T) { + // The claim the whole design rests on: find a record inside a rung and + // confirm the load figures for it are non-zero, i.e. computed over the + // same interval the CPU number came from rather than over a timer's own. + var found bool + for _, s := range samples { + if s.Rung < 0 { + continue + } + found = true + if s.Load.OfferedQPS <= 0 { + t.Errorf("rung %d record offered %v QPS; the schedule says otherwise", s.Rung, s.Load.OfferedQPS) + } + if s.RungQPS <= 0 { + t.Errorf("rung %d record has no nominal rate", s.Rung) + } + if s.Containers[RoleEnvoy].CPUCores <= 0 { + t.Errorf("rung %d record has load but no CPU: the panels would not line up", s.Rung) + } + } + if !found { + t.Error("no record fell inside a rung") + } + }) + + t.Run("WarmupIsMarkedNotDropped", func(t *testing.T) { + // A rung's first seconds are where the pool grows; they belong in the + // file, flagged, so exclusion is the analysis's decision and not the + // harness's. + var warm int + for _, s := range samples { + if s.Warmup { + warm++ + } + } + if warm == 0 { + t.Error("no record was flagged as warmup across three rungs with a 40ms warmup each") + } + }) + + t.Run("EnvoyAndRouterSectionsAreDifferenced", func(t *testing.T) { + s := samples[len(samples)-1] + if s.Envoy == nil { + t.Fatal("no envoy section") + } + if s.Envoy.Concurrency != 40 { + t.Errorf("envoy concurrency = %v, want the 40 the fixture reports", s.Envoy.Concurrency) + } + actor, ok := s.Envoy.Clusters[ActorClusterName] + if !ok { + t.Fatalf("no %s cluster in %v", ActorClusterName, s.Envoy.Clusters) + } + if actor.NewConnections != 10 { + t.Errorf("new connections = %v, want the 10 the counter advanced by", actor.NewConnections) + } + // The window's own ratio: 5000 requests over the 10 connections it + // opened. Present because this window did open some. + if actor.WindowRqPerCx == nil || *actor.WindowRqPerCx != 500 { + t.Errorf("window_rq_per_cx = %v, want 500 (5000 requests over 10 connections)", actor.WindowRqPerCx) + } + // The headline ratio is cumulative, so it reflects every connection the + // proxy has ever opened rather than only this window's. + if actor.RqPerCx <= 1 { + t.Errorf("rq_per_cx = %v, want well above 1: pooling is in force in the fixture", actor.RqPerCx) + } + if s.Router == nil || !s.Router.Measured { + t.Fatalf("router parking section = %+v, want it measured", s.Router) + } + if s.Router.ParkingActive != 12 { + t.Errorf("parking active = %v, want the fixture's 12", s.Router.ParkingActive) + } + }) + + t.Run("PortBudgetIsDerivedFromTheMeasuredRange", func(t *testing.T) { + s := samples[len(samples)-1] + if s.Ports.Available != 28232 { + t.Errorf("available ports = %d, want 28232", s.Ports.Available) + } + if s.Ports.ActiveConnections != 295 { + t.Errorf("active connections = %v, want Envoy's cx_active of 295", s.Ports.ActiveConnections) + } + if s.Ports.CircuitBreakerLimit >= s.Ports.Available { + t.Errorf("breaker limit %d is not below the %d-port budget; the kernel would run out first", + s.Ports.CircuitBreakerLimit, s.Ports.Available) + } + }) + + t.Run("EveryRequestTheClientSentWasAskedForByThePacer", func(t *testing.T) { + // 200+400+600 QPS held 150ms each = 30+60+90. + if got, want := client.count(), 180; got > want { + t.Errorf("client sent %d requests, more than the %d the ladder scheduled", got, want) + } + if client.count() == 0 { + t.Fatal("the ladder sent nothing") + } + }) + + if res.EnvoyConcurrency != 40 { + t.Errorf("result envoy concurrency = %v, want 40", res.EnvoyConcurrency) + } + if res.ClockSkewMs < 0 { + t.Errorf("clock skew = %vms; the sample cannot postdate the fetch that read it", res.ClockSkewMs) + } +} + +func TestRunnerFineSeriesCarriesNoResourceFields(t *testing.T) { + // Structural, not stylistic: the 1s series is not aligned to the resource + // panels, and the only durable way to stop someone plotting it against one + // is for the columns not to exist. + client := &fakeClient{connsInUse: 40, newConns: 2, reqsPerCx: 500} + sink := &memSink{} + ladder := LadderSpec{StartQPS: 200, StepQPS: 0, Rungs: 1, Hold: 120 * time.Millisecond} + r, _ := newTestRunner(t, client, sink, ladder) + + if _, err := r.Run(context.Background()); err != nil { + t.Fatalf("Run: %v", err) + } + _, fine := sink.all() + if len(fine) == 0 { + t.Fatal("no fine samples") + } + b, err := json.Marshal(fine[0]) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, forbidden := range []string{"cpu", "memory", "containers", "groups", "envoy", "ports"} { + if strings.Contains(string(b), forbidden) { + t.Errorf("fine sample carries %q: %s", forbidden, b) + } + } + if !strings.Contains(string(b), `"offered_qps"`) { + t.Errorf("fine sample lost the generator series it exists for: %s", b) + } +} + +func TestRunnerStopsTheLadderOnAFatalGuard(t *testing.T) { + // A rig-limited run must stop rather than keep emitting numbers that + // describe the load generator. + sink := &memSink{} + cfg := DefaultGuardConfig() + client := &fakeClient{ + connsInUse: int64(cfg.ClientConnectionCeiling) + 1, + newConns: 2, + reqsPerCx: 500, + } + // Long enough that finishing normally would take many seconds; the guard + // should cut it off inside the first rung. + ladder := LadderSpec{StartQPS: 100, StepQPS: 100, Rungs: 10, Hold: time.Second} + r, _ := newTestRunner(t, client, sink, ladder) + + res, err := r.Run(context.Background()) + + var rigErr *RigLimitedError + if !errors.As(err, &rigErr) { + t.Fatalf("Run returned %v, want a RigLimitedError", err) + } + if len(res.FatalTrips) == 0 { + t.Fatal("result carries no fatal trips to explain the stop") + } + if got := res.FatalTrips[0].Guard; got != GuardClientPorts { + t.Errorf("tripped %s, want %s", got, GuardClientPorts) + } + if len(res.Rungs) >= 10 { + t.Errorf("ran %d of 10 rungs; the guard did not stop the ladder", len(res.Rungs)) + } + + samples, _ := sink.all() + if len(samples) == 0 { + t.Fatal("no sample was written; the run directory would not explain why it stopped") + } + last := samples[len(samples)-1] + if !AnyFatal(last.Guards) { + t.Errorf("the last written sample does not carry the trip: %+v", last.Guards) + } +} + +func TestRunnerStopsOnAnInterrupt(t *testing.T) { + // Ctrl-C, or the Job being deleted: exit promptly and say it was cut short + // rather than reporting a complete ladder. + client := &fakeClient{connsInUse: 40, newConns: 2, reqsPerCx: 500} + sink := &memSink{} + ladder := LadderSpec{StartQPS: 100, StepQPS: 100, Rungs: 10, Hold: time.Second} + r, _ := newTestRunner(t, client, sink, ladder) + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(150*time.Millisecond, cancel) + + done := make(chan struct{}) + var res RunResult + var err error + go func() { + defer close(done) + res, err = r.Run(ctx) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Run did not return after its context was cancelled") + } + + if !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", err) + } + if !res.Interrupted { + t.Error("result does not record that the arm was interrupted") + } + if len(res.Rungs) >= 10 { + t.Errorf("ran %d of 10 rungs after a 150ms cancel", len(res.Rungs)) + } +} + +func TestRunnerRejectsAnIncompleteConfiguration(t *testing.T) { + cases := map[string]func(*Runner){ + "NoClient": func(r *Runner) { r.Client = nil }, + "NoSink": func(r *Runner) { r.Sink = nil }, + "NoWindows": func(r *Runner) { r.Windows = nil }, + "NoRungs": func(r *Runner) { r.Rungs = nil }, + } + for name, break_ := range cases { + t.Run(name, func(t *testing.T) { + r, _ := newTestRunner(t, &fakeClient{}, &memSink{}, + LadderSpec{StartQPS: 10, Rungs: 1, Hold: 10 * time.Millisecond}) + break_(r) + if _, err := r.Run(context.Background()); err == nil { + t.Fatal("Run accepted a runner missing one of its sources") + } + }) + } +} + +func TestRunnerFailsWhenTheAnchorIsAbsent(t *testing.T) { + // Expected right after an arm change: the router pod was replaced, so the + // anchor key no longer resolves and the caller must re-resolve it. Failing + // at prime is the point — an arm that silently measured nothing is worse. + r, _ := newTestRunner(t, &fakeClient{}, &memSink{}, + LadderSpec{StartQPS: 10, Rungs: 1, Hold: 10 * time.Millisecond}) + r.Windows.Client = &CadvisorClient{Fetch: staticFetch("# empty\n")} + + if _, err := r.Run(context.Background()); !errors.Is(err, ErrAnchorMissing) { + t.Fatalf("err = %v, want ErrAnchorMissing", err) + } +} + +func TestCollectorPeaksAreIndependentPerSeries(t *testing.T) { + // The two series read the same collector at different cadences. A shared + // high-water slot would let the 1s series reset the maximum the ~10s series + // is about to report, so in-flight — the one number that explains a port + // wall — would read as a fraction of its real peak. + c := NewCollector(&Schedule{}) + now := time.Now() + for i := 0; i < 5; i++ { + c.RecordDispatch(now, now) + } + // Drain back down to 1 before either read. Without this the live count + // still equals the peak, so the reset baseline equals what was consumed and + // a shared slot would pass the test it exists to fail. + for i := 0; i < 4; i++ { + c.RecordCompletion(now, now, OutcomeOK, 200) + } + + fine := c.FineStats(now.Add(-time.Second), now.Add(time.Second)) + aligned := c.Stats(now.Add(-time.Second), now.Add(time.Second)) + if fine.InFlightMax != 5 { + t.Errorf("fine series peak = %d, want 5", fine.InFlightMax) + } + if aligned.InFlightMax != 5 { + t.Errorf("aligned series peak = %d, want 5: the fine read consumed it", aligned.InFlightMax) + } +} + +func TestLadderSpecBuildsEvenRungs(t *testing.T) { + l := LadderSpec{StartQPS: 1000, StepQPS: 1000, Rungs: 16, Hold: 45 * time.Second, Warmup: 10 * time.Second} + rungs := l.Build() + if len(rungs) != 16 { + t.Fatalf("built %d rungs, want 16", len(rungs)) + } + if rungs[0].RateQPS != 1000 || rungs[15].RateQPS != 16000 { + t.Errorf("rates run %v..%v, want 1000..16000", rungs[0].RateQPS, rungs[15].RateQPS) + } + if l.PeakQPS() != 16000 { + t.Errorf("PeakQPS = %v, want 16000", l.PeakQPS()) + } + for i, r := range rungs { + if r.Index != i { + t.Errorf("rung %d has index %d", i, r.Index) + } + if r.Hold != 45*time.Second || r.Warmup != 10*time.Second { + t.Errorf("rung %d: hold=%v warmup=%v, want the same for every rung", i, r.Hold, r.Warmup) + } + if !r.StartAt.IsZero() { + t.Errorf("rung %d was built with a start time; only the pacer knows when a rung actually begins", i) + } + } +} + +func TestJSONLSinkWritesOneLinePerRecord(t *testing.T) { + dir := t.TempDir() + sink, err := OpenJSONLSink(dir) + if err != nil { + t.Fatalf("OpenJSONLSink: %v", err) + } + for i := 0; i < 3; i++ { + if err := sink.Sample(Sample{Arm: 40, Rung: i}); err != nil { + t.Fatalf("Sample: %v", err) + } + if err := sink.Fine(FineSample{Arm: 40, Rung: i}); err != nil { + t.Fatalf("Fine: %v", err) + } + } + + // Read before Close: unbuffered writes are what make a killed run's output + // usable, and a test that only reads after a clean Close would not notice + // buffering creeping in. + b, err := os.ReadFile(filepath.Join(dir, SamplesFile)) + if err != nil { + t.Fatalf("read samples: %v", err) + } + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + if len(lines) != 3 { + t.Fatalf("got %d sample lines before Close, want 3: output is being buffered", len(lines)) + } + var got Sample + if err := json.Unmarshal([]byte(lines[2]), &got); err != nil { + t.Fatalf("unmarshal line 3: %v", err) + } + if got.Rung != 2 { + t.Errorf("last line is rung %d, want 2", got.Rung) + } + if err := sink.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + fb, err := os.ReadFile(filepath.Join(dir, FineFile)) + if err != nil { + t.Fatalf("read fine: %v", err) + } + if n := len(strings.Split(strings.TrimSpace(string(fb)), "\n")); n != 3 { + t.Errorf("got %d fine lines, want 3", n) + } +} + +func TestStreamSinkTagsEveryLineSoTheStreamsCanBeSplitApart(t *testing.T) { + // The in-cluster Job's only usable output channel is stdout, shared by + // both series and the header. Without the tag, run.sh would have to guess + // the stream from field names. + var buf bytes.Buffer + s := NewStreamSink(&buf) + if err := s.Sample(Sample{Arm: 40, Rung: 2}); err != nil { + t.Fatalf("Sample: %v", err) + } + if err := s.Fine(FineSample{Arm: 40, Rung: 2}); err != nil { + t.Fatalf("Fine: %v", err) + } + if err := s.Header(RunHeader{Name: "routercap"}); err != nil { + t.Fatalf("Header: %v", err) + } + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + want := []string{StreamSample, StreamFine, StreamHeader} + if len(lines) != len(want) { + t.Fatalf("got %d lines, want %d", len(lines), len(want)) + } + for i, w := range want { + var got struct { + Stream string `json:"stream"` + Record json.RawMessage `json:"record"` + } + if err := json.Unmarshal([]byte(lines[i]), &got); err != nil { + t.Fatalf("line %d: %v", i, err) + } + if got.Stream != w { + t.Errorf("line %d is stream %q, want %q", i, got.Stream, w) + } + if len(got.Record) == 0 { + t.Errorf("line %d carries no record", i) + } + } +} + +func TestMultiSinkKeepsGoingWhenOneSinkFails(t *testing.T) { + // A full disk on the laptop copy must not cost us the stdout copy, which in + // an in-cluster run is the only copy that leaves the pod. + var buf bytes.Buffer + stream := NewStreamSink(&buf) + m := MultiSink{failingSink{}, stream} + + if err := m.Sample(Sample{Arm: 40}); err == nil { + t.Error("MultiSink hid a sink's write failure") + } + if err := m.Fine(FineSample{Arm: 40}); err == nil { + t.Error("MultiSink hid a sink's write failure") + } + if n := len(strings.Split(strings.TrimSpace(buf.String()), "\n")); n != 2 { + t.Errorf("the healthy sink got %d lines, want 2: a failing sink stopped the fan-out", n) + } +} + +type failingSink struct{} + +func (failingSink) Sample(Sample) error { return errors.New("disk full") } +func (failingSink) Fine(FineSample) error { return errors.New("disk full") } + +func TestWriteHeaderRecordsTheOrderingTheDesignClaims(t *testing.T) { + dir := t.TempDir() + h := RunHeader{ + Name: "routercap", + StartedAt: time.Now(), + PortRange: PortRange{Low: 32768, High: 60999, Source: PortRangeMeasured}, + CircuitBreakerLimit: 20000, + ExtProcMaxRequests: 20000, + ArmCores: []int{10, 20, 40, 70}, + Actors: 100, + Ladder: LadderSpec{StartQPS: 1000, StepQPS: 1000, Rungs: 16, Hold: 45 * time.Second}, + Guards: DefaultGuardConfig(), + Caveats: StandingCaveats(), + } + if err := WriteHeader(dir, h); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + + b, err := os.ReadFile(filepath.Join(dir, HeaderFile)) + if err != nil { + t.Fatalf("read header: %v", err) + } + var got RunHeader + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + // The design's central ordering claim, checkable from the output alone. + if got.CircuitBreakerLimit >= got.PortRange.Size() { + t.Errorf("breaker limit %d is not below the %d-port range; the kernel would exhaust before Envoy counted an overflow", + got.CircuitBreakerLimit, got.PortRange.Size()) + } + if got.ExtProcMaxRequests >= got.PortRange.Size() { + t.Errorf("ext_proc limit %d is not below the %d-port range", got.ExtProcMaxRequests, got.PortRange.Size()) + } + if got.PortRange.Source != PortRangeMeasured { + t.Errorf("port range source = %q, want it to record how the range was obtained", got.PortRange.Source) + } + if got.Guards.LoadgenCPUUtilization != DefaultGuardConfig().LoadgenCPUUtilization { + t.Error("guard thresholds did not survive the round trip; a loosened threshold would be invisible to a reader") + } + if len(got.Caveats) == 0 { + t.Error("header carries no caveats") + } +} diff --git a/internal/benchmarking/routercap/sender.go b/internal/benchmarking/routercap/sender.go new file mode 100644 index 000000000..f4db7099b --- /dev/null +++ b/internal/benchmarking/routercap/sender.go @@ -0,0 +1,224 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The HTTP client that puts one ping through the router, addressing an actor by Host +// header and counting connections as they are dialed. + +package routercap + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "sync/atomic" + "time" + + gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "github.com/google/uuid" + "google.golang.org/protobuf/proto" +) + +// pingPath is the glutton actor's echo endpoint. +const pingPath = "/ping" + +// countingDialer wraps the transport's dialer so the generator can report on +// its own connection behavior. If keep-alive silently stops working, the +// generator's socket-per-request cliff would look exactly like the router's. +type countingDialer struct { + inner *net.Dialer + opened atomic.Int64 + live atomic.Int64 +} + +func (d *countingDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + c, err := d.inner.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + d.opened.Add(1) + d.live.Add(1) + return &countedConn{Conn: c, dialer: d}, nil +} + +type countedConn struct { + net.Conn + dialer *countingDialer + closed atomic.Bool +} + +func (c *countedConn) Close() error { + if c.closed.CompareAndSwap(false, true) { + c.dialer.live.Add(-1) + } + return c.Conn.Close() +} + +// Sender issues pings to the router, round-robin across the warm actor pool +// and, when the router runs more than one replica, round-robin across the +// replicas too. +type Sender struct { + client *http.Client + dialer *countingDialer + urls []string + actors []Actor + next atomic.Uint64 + lastOpn int64 + + // dispatched counts requests handed to the transport, used with the dialer's + // connection count to derive requests-per-connection. + dispatched atomic.Int64 + lastDisp int64 +} + +// SenderConfig configures the generator's transport. +type SenderConfig struct { + // RouterURLs are the router pods' plaintext HTTP listeners, e.g. + // http://10.0.0.5:8080, one per replica, addressed by pod IP so no Service, + // kube-proxy hop or DNS lookup sits inside the measured path. Requests + // round-robin over the list, with a per-host connection pool per replica. + RouterURLs []string + Actors []Actor + // MaxConnections sizes the idle pool and must be at least the run's + // in-flight cap: Go closes idle connections above MaxIdleConnsPerHost, so + // a smaller pool churns connections at exactly the peak load. + MaxConnections int + // RequestTimeout bounds one ping. Timeouts are counted as failures and + // contribute their full latency to the percentiles rather than vanishing. + RequestTimeout time.Duration +} + +// NewSender builds the generator's HTTP client. +func NewSender(cfg SenderConfig) (*Sender, error) { + if len(cfg.RouterURLs) == 0 || cfg.RouterURLs[0] == "" { + return nil, fmt.Errorf("at least one router URL is required") + } + if len(cfg.Actors) == 0 { + return nil, fmt.Errorf("sender needs at least one warm actor") + } + maxConns := cfg.MaxConnections + if maxConns <= 0 { + maxConns = 1024 + } + timeout := cfg.RequestTimeout + if timeout <= 0 { + timeout = 30 * time.Second + } + + d := &countingDialer{inner: &net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}} + tr := &http.Transport{ + DialContext: d.DialContext, + MaxIdleConns: maxConns, + IdleConnTimeout: 120 * time.Second, + // Sized to the in-flight cap for the reason on MaxConnections. + MaxIdleConnsPerHost: maxConns, + // Unbounded on purpose: a per-host cap would make the transport queue + // requests internally, turning the open loop closed. The pacer's + // in-flight cap is the only concurrency bound, and it sheds visibly. + MaxConnsPerHost: 0, + // HTTP/1.1 only: the upstream hop and the real client are HTTP/1.1, + // and h2 would multiplex requests onto a handful of connections. + ForceAttemptHTTP2: false, + DisableCompression: true, + DisableKeepAlives: false, + } + urls := make([]string, len(cfg.RouterURLs)) + for i, u := range cfg.RouterURLs { + urls[i] = u + pingPath + } + return &Sender{ + client: &http.Client{Transport: tr, Timeout: timeout}, + dialer: d, + urls: urls, + actors: cfg.Actors, + }, nil +} + +// Send issues one ping and classifies the result. It is the SendFunc the pacer +// calls; the pacer owns timing, this owns the request. +func (s *Sender) Send(ctx context.Context) (Outcome, int) { + n := s.next.Add(1) - 1 + a := s.actors[int(n)%len(s.actors)] + // One counter drives both rotations, so the actor->replica mapping is + // sticky whenever the replica count divides the actor count — deliberate: + // each replica serves a fixed share of actors on its own warm pool. + url := s.urls[int(n)%len(s.urls)] + s.dispatched.Add(1) + + message := uuid.NewString() + body, err := proto.Marshal(&gluttonpb.PingRequest{Message: message}) + if err != nil { + return OutcomeBadBody, 0 + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return OutcomeTransportError, 0 + } + // The router routes on Host, not on the URL. The pod-IP URL only decides + // which socket the bytes go down. + req.Host = a.Host + req.Header.Set("Content-Type", "application/x-protobuf") + + resp, err := s.client.Do(req) + if err != nil { + return OutcomeTransportError, 0 + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + // A body that could not be drained also leaves the connection + // unreusable, which is why the read happens even on the error paths. + return OutcomeTransportError, resp.StatusCode + } + if resp.StatusCode >= 400 { + return OutcomeHTTPError, resp.StatusCode + } + + pong := &gluttonpb.PingResponse{} + if err := proto.Unmarshal(respBody, pong); err != nil { + return OutcomeBadBody, resp.StatusCode + } + if pong.Message != message { + // Something answered, but not the actor addressed: a misroute that + // returns 200 must not score as a success. + return OutcomeBadBody, resp.StatusCode + } + return OutcomeOK, resp.StatusCode +} + +// Stats returns the transport's behavior since the last call, so consecutive +// calls partition the run into non-overlapping windows the same way the counter +// deltas elsewhere do. +func (s *Sender) Stats() ClientStats { + opened := s.dialer.opened.Load() + dispatched := s.dispatched.Load() + newConns := float64(opened - s.lastOpn) + reqs := float64(dispatched - s.lastDisp) + s.lastOpn, s.lastDisp = opened, dispatched + + cs := ClientStats{NewConnections: newConns, ConnectionsInUse: s.dialer.live.Load()} + if newConns > 0 { + cs.RequestsPerConnection = reqs / newConns + } + return cs +} + +// CloseIdleConnections releases the pool. Called at teardown so the generator +// does not leave thousands of sockets in TIME_WAIT for the next arm's Job. +func (s *Sender) CloseIdleConnections() { + s.client.CloseIdleConnections() +} diff --git a/internal/benchmarking/routercap/sender_test.go b/internal/benchmarking/routercap/sender_test.go new file mode 100644 index 000000000..b27c29c2c --- /dev/null +++ b/internal/benchmarking/routercap/sender_test.go @@ -0,0 +1,252 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for the sender's addressing, outcome classification and connection reuse. + +package routercap + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + gluttonpb "github.com/agent-substrate/substrate/internal/proto/glutton" + "google.golang.org/protobuf/proto" +) + +// fakeActor stands in for the router plus a glutton actor: it echoes the ping +// message back, and records the Host header it was addressed with. +type fakeActor struct { + mu sync.Mutex + hosts []string + status int + // corrupt makes the echo come back with a different message, standing in + // for a misroute that still returns 200. + corrupt bool +} + +func (f *fakeActor) handler(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.hosts = append(f.hosts, r.Host) + st, corrupt := f.status, f.corrupt + f.mu.Unlock() + + if st != 0 && st != http.StatusOK { + http.Error(w, "upstream connect error or disconnect/reset before headers", st) + return + } + ping := &gluttonpb.PingRequest{} + if err := proto.Unmarshal(body, ping); err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + msg := ping.Message + if corrupt { + msg = "not-the-message-you-sent" + } + out, _ := proto.Marshal(&gluttonpb.PingResponse{Message: msg}) + w.Header().Set("Content-Type", "application/x-protobuf") + w.Write(out) +} + +func (f *fakeActor) seenHosts() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.hosts...) +} + +func testActors(n int) []Actor { + out := make([]Actor, n) + for i := range out { + name := string(rune('a' + i)) + out[i] = Actor{Atespace: "routercap", Name: name, Host: name + ".routercap." + actorDomain} + } + return out +} + +func newTestSender(t *testing.T, url string, actors []Actor) *Sender { + t.Helper() + s, err := NewSender(SenderConfig{RouterURLs: []string{url}, Actors: actors, MaxConnections: 64, RequestTimeout: 5 * time.Second}) + if err != nil { + t.Fatalf("NewSender: %v", err) + } + t.Cleanup(s.CloseIdleConnections) + return s +} + +func TestSenderAddressesActorsByHostHeader(t *testing.T) { + // The router routes on Host; the URL only picks the socket. If this ever + // stops holding, every request lands on one actor and the run measures a + // single worker pod. + fa := &fakeActor{} + srv := httptest.NewServer(http.HandlerFunc(fa.handler)) + defer srv.Close() + + actors := testActors(3) + s := newTestSender(t, srv.URL, actors) + for i := 0; i < 6; i++ { + if out, st := s.Send(context.Background()); out != OutcomeOK { + t.Fatalf("send %d: outcome %s status %d", i, out, st) + } + } + + hosts := fa.seenHosts() + if len(hosts) != 6 { + t.Fatalf("server saw %d requests, want 6", len(hosts)) + } + counts := map[string]int{} + for _, h := range hosts { + counts[h]++ + } + if len(counts) != 3 { + t.Fatalf("load hit %d distinct actors, want 3: %v", len(counts), counts) + } + for _, a := range actors { + if counts[a.Host] != 2 { + t.Errorf("actor %s got %d of 6 requests, want an even 2", a.Host, counts[a.Host]) + } + } +} + +func TestSenderClassifiesOutcomes(t *testing.T) { + cases := []struct { + name string + setup func(*fakeActor) + want Outcome + wantSt int + checkSt bool + }{ + {"OK", func(f *fakeActor) {}, OutcomeOK, 200, true}, + { + // Envoy's shed response when a circuit breaker trips. It must land + // in the latency distribution, not vanish from it. + "CircuitBreakerShed", + func(f *fakeActor) { f.status = http.StatusServiceUnavailable }, + OutcomeHTTPError, 503, true, + }, + { + // A 200 carrying someone else's payload is the one failure a + // status-code-only harness scores as a success. + "MisrouteWithA200", + func(f *fakeActor) { f.corrupt = true }, + OutcomeBadBody, 200, true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fa := &fakeActor{} + tc.setup(fa) + srv := httptest.NewServer(http.HandlerFunc(fa.handler)) + defer srv.Close() + + s := newTestSender(t, srv.URL, testActors(1)) + got, st := s.Send(context.Background()) + if got != tc.want { + t.Errorf("outcome = %s, want %s", got, tc.want) + } + if tc.checkSt && st != tc.wantSt { + t.Errorf("status = %d, want %d", st, tc.wantSt) + } + }) + } +} + +func TestSenderReportsATransportFailureWithNoStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + s := newTestSender(t, srv.URL, testActors(1)) + srv.Close() // nothing is listening now + + got, st := s.Send(context.Background()) + if got != OutcomeTransportError { + t.Errorf("outcome = %s, want %s", got, OutcomeTransportError) + } + if st != 0 { + t.Errorf("status = %d, want 0: there was no response to take one from", st) + } +} + +func TestSenderReusesConnections(t *testing.T) { + // The property the whole rig depends on. Without keep-alive the generator + // burns one source port per request and hits its own ceiling long before + // the router hits anything. + fa := &fakeActor{} + srv := httptest.NewServer(http.HandlerFunc(fa.handler)) + defer srv.Close() + + s := newTestSender(t, srv.URL, testActors(4)) + for i := 0; i < 50; i++ { + if out, _ := s.Send(context.Background()); out != OutcomeOK { + t.Fatalf("send %d failed with %s", i, out) + } + } + + // Serial sends: one connection carries all fifty. + cs := s.Stats() + if cs.NewConnections != 1 { + t.Errorf("NewConnections = %v, want 1 for 50 serial requests", cs.NewConnections) + } + if cs.RequestsPerConnection != 50 { + t.Errorf("RequestsPerConnection = %v, want 50", cs.RequestsPerConnection) + } + if cs.ConnectionsInUse != 1 { + t.Errorf("ConnectionsInUse = %d, want 1", cs.ConnectionsInUse) + } +} + +func TestSenderStatsPartitionTheRun(t *testing.T) { + // Consecutive Stats calls must not double-count, or the keep-alive guard + // would read connection churn that already happened in an earlier window. + fa := &fakeActor{} + srv := httptest.NewServer(http.HandlerFunc(fa.handler)) + defer srv.Close() + + s := newTestSender(t, srv.URL, testActors(1)) + for i := 0; i < 10; i++ { + s.Send(context.Background()) + } + first := s.Stats() + for i := 0; i < 10; i++ { + s.Send(context.Background()) + } + second := s.Stats() + + if first.NewConnections != 1 { + t.Errorf("first window NewConnections = %v, want 1", first.NewConnections) + } + if second.NewConnections != 0 { + t.Errorf("second window NewConnections = %v, want 0: the connection was opened in the first window", second.NewConnections) + } + // Zero new connections is perfect reuse, not a ratio of zero; the + // keep-alive guard skips this case rather than reading it as a failure. + if second.RequestsPerConnection != 0 { + t.Errorf("RequestsPerConnection = %v, want 0 when nothing was dialed", second.RequestsPerConnection) + } + if AnyFatal(DefaultGuardConfig().Check(&Sample{Client: second, Containers: map[string]ContainerUsage{}})) { + t.Error("a window with perfect connection reuse tripped a fatal guard") + } +} + +func TestNewSenderRejectsAnEmptyPool(t *testing.T) { + if _, err := NewSender(SenderConfig{RouterURLs: []string{"http://10.0.0.1:8080"}}); err == nil { + t.Fatal("NewSender accepted a sender with no actors to address") + } + if _, err := NewSender(SenderConfig{Actors: testActors(1)}); err == nil { + t.Fatal("NewSender accepted a sender with no router URL") + } +} diff --git a/internal/benchmarking/routercap/spans.go b/internal/benchmarking/routercap/spans.go new file mode 100644 index 000000000..99bc49c88 --- /dev/null +++ b/internal/benchmarking/routercap/spans.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Splitting the average request into hops: how much of it is the generator +// getting to Envoy, Envoy's own handling, the sidecar callout, and the worker. + +package routercap + +// LatencySpans divides the average request across the path +// +// client -> Envoy -> sidecar (resume the actor) -> worker -> back +// +// into four spans that do not overlap and sum to the whole: +// +// BeforeEnvoyMs client mean latency - time inside Envoy (residual) +// EnvoyInternalMs time inside Envoy - worker hop - sidecar hop (residual) +// SidecarMs the sidecar's route-duration histogram (measured) +// WorkerMs Envoy's upstream_rq_time on the actor cluster (measured) +// +// Everything here is a mean, never a percentile: percentiles do not decompose +// across hops, so TotalClientMs is not the p50 quoted elsewhere. +// +// BeforeEnvoyMs is the rig's contribution (generator queueing, dial, handshake), +// timed from scheduled send. EnvoyInternalMs is Envoy's own work plus the +// microsecond-scale loopback wire to the sidecar, which cannot be separated. +// +// Envoy stores request times as whole milliseconds, so both Envoy-derived +// quantities are biased low, which biases the two residuals up; +// ResolutionMsShare bounds that as a fraction of the total. Envoy records no +// rq_time for the ext_proc cluster (every stream ends in a reset), so the +// sidecar's own histogram is the only instrument for that hop. See +// benchmarking/routercap/RESULTS.md. +type LatencySpans struct { + // Measured is false when Envoy reported no request-time samples for the + // window, which makes every field below meaningless rather than zero. + Measured bool `json:"measured"` + // SidecarMeasured is false when the sidecar exposed no route-duration + // series. The other three spans still hold; SidecarMs is then zero and its + // time sits inside EnvoyInternalMs. + SidecarMeasured bool `json:"sidecar_measured"` + + BeforeEnvoyMs float64 `json:"before_envoy_ms"` + EnvoyInternalMs float64 `json:"envoy_internal_ms"` + SidecarMs float64 `json:"sidecar_ms"` + WorkerMs float64 `json:"worker_ms"` + + // ResumeMs is the control-plane round trip the sidecar makes to wake the + // actor; it is not part of the stack and must never be added to the four + // spans. The resume nests inside the route per request, not in the means — + // the two histograms have different denominators, so ResumeMs can + // legitimately exceed SidecarMs; do not "correct" it or assert the + // inequality. + ResumeMs float64 `json:"resume_ms"` + + // TotalClientMs and InEnvoyMs are the two totals the residuals are taken + // from, carried so a reader can redo the subtraction. + TotalClientMs float64 `json:"total_client_ms"` + InEnvoyMs float64 `json:"in_envoy_ms"` + + // ResolutionMsShare is Envoy's whole-millisecond floor as a fraction of the + // mean request: 0.01 means the split can be read as it stands, 0.4 means it + // is mostly an artifact of the instrument. + ResolutionMsShare float64 `json:"resolution_ms_share"` + + // The four spans come from four instruments in three processes, each with + // its own request-count denominator. CountSpread is the largest of the four + // counts divided by the smallest, minus one: 0.01 means the arithmetic is + // sound, 0.5 means the spans describe substantially different populations. + ClientRequests float64 `json:"client_requests"` + InEnvoyRequests float64 `json:"in_envoy_requests"` + SidecarRequests float64 `json:"sidecar_requests"` + WorkerRequests float64 `json:"worker_requests"` + CountSpread float64 `json:"count_spread"` +} + +// latencySpans derives the breakdown for one window, returning nil when Envoy +// gave no request-time samples to divide. Residuals are not clamped: a negative +// residual is a fact about instrument disagreement and belongs in the record. +func latencySpans(load GenStats, e *EnvoyDelta, r *RouterDelta) *LatencySpans { + if e == nil || e.InEnvoySamples <= 0 { + return nil + } + s := LatencySpans{ + Measured: true, + InEnvoyMs: e.MeanInEnvoyMs, + InEnvoyRequests: e.InEnvoySamples, + TotalClientMs: load.Latency.MeanMs, + ClientRequests: float64(load.Latency.Count), + } + if actor, ok := e.Clusters[ActorClusterName]; ok { + s.WorkerMs, s.WorkerRequests = actor.MeanRqTimeMs, actor.RqTimeSamples + } + if r != nil && r.RouteMeasured { + s.SidecarMeasured = true + s.SidecarMs, s.SidecarRequests = r.MeanRouteMs, r.RouteCalls + s.ResumeMs = r.MeanResumeMs + } + s.EnvoyInternalMs = s.InEnvoyMs - s.WorkerMs - s.SidecarMs + s.BeforeEnvoyMs = s.TotalClientMs - s.InEnvoyMs + s.CountSpread = countSpread(s.ClientRequests, s.InEnvoyRequests, s.SidecarRequests, s.WorkerRequests) + if s.TotalClientMs > 0 { + s.ResolutionMsShare = envoyResolutionMs / s.TotalClientMs + } + return &s +} + +// envoyResolutionMs is the granularity Envoy stores a request time at. Not a +// tunable: it is the unit of the counter. +const envoyResolutionMs = 1.0 + +// countSpread reports how far apart the non-zero denominators are, as a +// fraction of the smallest. Zeros are skipped: an unmeasured span is already +// flagged by its own Measured field. +func countSpread(counts ...float64) float64 { + lo, hi := 0.0, 0.0 + for _, c := range counts { + if c <= 0 { + continue + } + if lo == 0 || c < lo { + lo = c + } + if c > hi { + hi = c + } + } + if lo == 0 { + return 0 + } + return hi/lo - 1 +} diff --git a/internal/benchmarking/routercap/spans_test.go b/internal/benchmarking/routercap/spans_test.go new file mode 100644 index 000000000..d21d1073d --- /dev/null +++ b/internal/benchmarking/routercap/spans_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for the per-hop breakdown of the average request. + +package routercap + +import ( + "math" + "testing" +) + +// spanInputs builds the three sources of one window's breakdown: a client mean +// of 100ms, 80ms of it inside Envoy, a 16ms worker hop and a 1.25ms sidecar hop +// of which 1.2ms is the resume. That leaves 20ms before Envoy and 62.75ms +// inside it — roughly the shape the 8-core arm actually produced. +func spanInputs() (GenStats, *EnvoyDelta, *RouterDelta) { + load := GenStats{Latency: LatencyStats{Count: 10000, MeanMs: 100}} + envoy := &EnvoyDelta{ + MeanInEnvoyMs: 80, + InEnvoySamples: 10000, + Clusters: map[string]ClusterDelta{ + ActorClusterName: {MeanRqTimeMs: 16, RqTimeSamples: 10000}, + // The ext_proc cluster reports no rq_time, as on the live proxy. + ExtProcClusterName: {Requests: 10000}, + }, + } + router := &RouterDelta{ + Measured: true, RouteMeasured: true, + MeanRouteMs: 1.25, RouteCalls: 10000, MeanResumeMs: 1.2, + } + return load, envoy, router +} + +func TestLatencySpansSumToTheWhole(t *testing.T) { + s := latencySpans(spanInputs()) + if s == nil { + t.Fatal("latencySpans returned nil on a fully measured window") + } + // The defining property: four non-overlapping spans covering the whole + // request. If this drifts, the stacked chart is drawing a fiction. + sum := s.BeforeEnvoyMs + s.EnvoyInternalMs + s.SidecarMs + s.WorkerMs + if math.Abs(sum-s.TotalClientMs) > 1e-9 { + t.Errorf("spans sum to %v ms, want the client mean %v ms", sum, s.TotalClientMs) + } + if s.BeforeEnvoyMs != 20 { + t.Errorf("BeforeEnvoyMs = %v, want 20", s.BeforeEnvoyMs) + } + if s.WorkerMs != 16 || s.SidecarMs != 1.25 { + t.Errorf("worker/sidecar = %v/%v, want 16/1.25", s.WorkerMs, s.SidecarMs) + } + if want := 62.75; math.Abs(s.EnvoyInternalMs-want) > 1e-9 { + t.Errorf("EnvoyInternalMs = %v, want %v", s.EnvoyInternalMs, want) + } +} + +// TestLatencySpansKeepsResumeOutOfTheStack pins the nesting. The resume happens +// inside the sidecar's handler, so counting it as a fifth span would attribute +// its time twice and inflate the total past what any client saw. +func TestLatencySpansKeepsResumeOutOfTheStack(t *testing.T) { + s := latencySpans(spanInputs()) + if s.ResumeMs != 1.2 { + t.Errorf("ResumeMs = %v, want 1.2", s.ResumeMs) + } + sum := s.BeforeEnvoyMs + s.EnvoyInternalMs + s.SidecarMs + s.WorkerMs + s.ResumeMs + if math.Abs(sum-s.TotalClientMs) < 1e-9 { + t.Error("the four spans plus the resume still sum to the total, so the resume is being double-counted") + } +} + +// TestLatencySpansCarriesAResumeLargerThanTheHandler pins a shape that looks +// broken and is not: the route and parking histograms cover different +// populations, so under shedding ResumeMs can exceed SidecarMs. The record +// must carry it as measured. +func TestLatencySpansCarriesAResumeLargerThanTheHandler(t *testing.T) { + load, envoy, router := spanInputs() + // The shape of one observed window: 27397 routes against 16326 resumes, + // mean route 521ms, mean resume 874ms. + router.RouteCalls, router.MeanRouteMs = 27397, 520.67 + router.MeanResumeMs = 873.69 + + s := latencySpans(load, envoy, router) + if s.ResumeMs <= s.SidecarMs { + t.Fatalf("ResumeMs %v, SidecarMs %v: the fixture no longer covers the case", s.ResumeMs, s.SidecarMs) + } + if s.SidecarMs != 520.67 { + t.Errorf("SidecarMs = %v, want the route mean 520.67 unaltered", s.SidecarMs) + } + // The four spans still partition the request. The resume is outside them, so + // its size relative to the sidecar cannot break the sum. + sum := s.BeforeEnvoyMs + s.EnvoyInternalMs + s.SidecarMs + s.WorkerMs + if math.Abs(sum-s.TotalClientMs) > 1e-9 { + t.Errorf("spans sum to %v ms, want the client mean %v ms", sum, s.TotalClientMs) + } +} + +// TestLatencySpansWithoutTheSidecarInstrument covers an older router image +// with no route-duration series. The sidecar's time folds into +// EnvoyInternalMs rather than being silently deducted as zero. +func TestLatencySpansWithoutTheSidecarInstrument(t *testing.T) { + load, envoy, _ := spanInputs() + s := latencySpans(load, envoy, &RouterDelta{Measured: true}) + if s.SidecarMeasured { + t.Error("SidecarMeasured = true with no route-duration series") + } + if s.SidecarMs != 0 { + t.Errorf("SidecarMs = %v, want 0 when unmeasured", s.SidecarMs) + } + if want := 64.0; s.EnvoyInternalMs != want { + t.Errorf("EnvoyInternalMs = %v, want %v (the sidecar's 1.25ms stays fused in)", s.EnvoyInternalMs, want) + } + sum := s.BeforeEnvoyMs + s.EnvoyInternalMs + s.SidecarMs + s.WorkerMs + if math.Abs(sum-s.TotalClientMs) > 1e-9 { + t.Errorf("spans sum to %v ms, want %v", sum, s.TotalClientMs) + } +} + +func TestLatencySpansIsNilWithoutEnvoyRequestTimes(t *testing.T) { + load, _, router := spanInputs() + if s := latencySpans(load, nil, router); s != nil { + t.Error("latencySpans returned a breakdown with no Envoy delta") + } + // An idle window: Envoy answered nothing, so there is no mean to divide. + // Nil, not a record full of zeros that would plot as a real observation. + if s := latencySpans(load, &EnvoyDelta{}, router); s != nil { + t.Error("latencySpans returned a breakdown from zero request-time samples") + } +} + +// TestLatencySpansReportsDisagreeingDenominators covers the case that makes the +// breakdown untrustworthy: the four instruments describing different sets of +// requests. It is reported, not corrected — the arithmetic is still the best +// available, and the reader needs to know how far to trust it. +func TestLatencySpansReportsDisagreeingDenominators(t *testing.T) { + load, envoy, router := spanInputs() + router.RouteCalls = 5000 // the sidecar saw half as many requests + + s := latencySpans(load, envoy, router) + if want := 1.0; math.Abs(s.CountSpread-want) > 1e-9 { + t.Errorf("CountSpread = %v, want %v (10000 against 5000)", s.CountSpread, want) + } + + // Agreement reads as near zero, so a threshold can be applied to either. + router.RouteCalls = 9900 + if s := latencySpans(load, envoy, router); s.CountSpread > 0.02 { + t.Errorf("CountSpread = %v on denominators agreeing to 1%%", s.CountSpread) + } +} + +// TestLatencySpansCarriesTheResolutionFloor pins the readability number: +// Envoy stores request times as whole milliseconds. At a 2.5ms mean the +// truncation is worth 40% of the request; at 100ms, 1%. +func TestLatencySpansCarriesTheResolutionFloor(t *testing.T) { + load, envoy, router := spanInputs() + if s := latencySpans(load, envoy, router); math.Abs(s.ResolutionMsShare-0.01) > 1e-9 { + t.Errorf("ResolutionMsShare = %v at a 100ms mean, want 0.01", s.ResolutionMsShare) + } + + load.Latency.MeanMs = 2.5 + if s := latencySpans(load, envoy, router); math.Abs(s.ResolutionMsShare-0.4) > 1e-9 { + t.Errorf("ResolutionMsShare = %v at a 2.5ms mean, want 0.4", s.ResolutionMsShare) + } + + // An idle window divides by nothing rather than reporting an infinite floor. + load.Latency.MeanMs = 0 + if s := latencySpans(load, envoy, router); s.ResolutionMsShare != 0 { + t.Errorf("ResolutionMsShare = %v with no client mean, want 0", s.ResolutionMsShare) + } +} + +// TestLatencySpansDoesNotClampANegativeResidual keeps a broken window visibly +// broken. If the instruments disagree enough to push a residual below zero, +// silently flooring it at zero would turn a detectable fault into a plausible +// looking chart. +func TestLatencySpansDoesNotClampANegativeResidual(t *testing.T) { + load, envoy, router := spanInputs() + envoy.Clusters[ActorClusterName] = ClusterDelta{MeanRqTimeMs: 200, RqTimeSamples: 10000} + + s := latencySpans(load, envoy, router) + if s.EnvoyInternalMs >= 0 { + t.Errorf("EnvoyInternalMs = %v, want negative: the worker hop alone exceeded the in-Envoy total", + s.EnvoyInternalMs) + } +} diff --git a/internal/benchmarking/routercap/stats.go b/internal/benchmarking/routercap/stats.go new file mode 100644 index 000000000..d57cfcda7 --- /dev/null +++ b/internal/benchmarking/routercap/stats.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package routercap measures how much offered load one atenet-router pod +// absorbs, as a time series rather than a single number. The four headline +// series — offered load, client latency, router CPU, router memory — share +// the same wall-clock window; see window.go for how that is arranged. +package routercap + +import ( + "math" + "sort" + "time" +) + +// LatencyStats is an exact summary of a set of latencies — exact because the +// tail must distinguish 200ms from 2s, which a histogram that wide cannot. +// p95 is the tail metric; over a few-thousand-sample window p99 and max are +// mostly sampling noise. +type LatencyStats struct { + Count int `json:"count"` + P50Ms float64 `json:"p50_ms"` + P95Ms float64 `json:"p95_ms"` + MeanMs float64 `json:"mean_ms"` +} + +// summarize computes exact percentiles over ds. It sorts in place, so callers +// that still need the original order must pass a copy. +func summarize(ds []time.Duration) LatencyStats { + s := LatencyStats{Count: len(ds)} + if len(ds) == 0 { + return s + } + sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] }) + var total float64 + for _, d := range ds { + total += msOf(d) + } + s.MeanMs = total / float64(len(ds)) + s.P50Ms = quantileSorted(ds, 0.50) + s.P95Ms = quantileSorted(ds, 0.95) + return s +} + +// quantileSorted returns the q-quantile of an already-sorted slice using the +// nearest-rank method: the smallest value at or above the q fraction. Every +// reported value is therefore a latency some request actually experienced. +func quantileSorted(sorted []time.Duration, q float64) float64 { + if len(sorted) == 0 { + return 0 + } + rank := int(math.Ceil(q * float64(len(sorted)))) + if rank < 1 { + rank = 1 + } + if rank > len(sorted) { + rank = len(sorted) + } + return msOf(sorted[rank-1]) +} + +func msOf(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) } diff --git a/internal/benchmarking/routercap/window.go b/internal/benchmarking/routercap/window.go new file mode 100644 index 000000000..4db26d68a --- /dev/null +++ b/internal/benchmarking/routercap/window.go @@ -0,0 +1,185 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The cAdvisor-defined measurement window: waits for the anchor container's timestamp +// to advance, which is what makes [t0,t1) a real interval rather than a guess. + +package routercap + +import ( + "context" + "errors" + "fmt" + "time" +) + +// ErrAnchorMissing means the anchor container was absent from a scrape. It is +// expected across an arm change, when the router pod is replaced, and the +// caller is supposed to re-resolve the pod rather than treat it as a failure. +var ErrAnchorMissing = errors.New("anchor container absent from cadvisor scrape") + +// Window is one measurement interval, with its bounds taken from cAdvisor +// rather than from a local timer. The kubelet housekeeps roughly every 10s, so +// the driver waits for the anchor container's measurement to change and then +// asks every other source for the interval that measurement covers. +type Window struct { + // T0 and T1 are the anchor container's cAdvisor timestamps: the instants + // its cumulative counters were read. + T0, T1 time.Time + // Prev and Cur are the scrapes those timestamps came from. + Prev, Cur CadvisorScrape + // Polls is how many fetches it took for the anchor timestamp to advance. + // One means the kubelet moved faster than the poll interval and the + // resolution is poll-limited rather than kubelet-limited. + Polls int +} + +// Duration is the length of the interval. +func (w Window) Duration() time.Duration { return w.T1.Sub(w.T0) } + +// Mid is the interval midpoint, the natural x value when a chart has to draw +// an interval as a point. +func (w Window) Mid() time.Time { return w.T0.Add(w.Duration() / 2) } + +// Usage computes each requested container's usage over the window; containers +// absent from either scrape land in missing rather than being silently +// omitted. Each container's rate uses its own pair of cAdvisor timestamps, and +// spread reports the largest disagreement with the anchor's interval. +func (w Window) Usage(keys []ContainerKey) (usage map[ContainerKey]ContainerUsage, spread time.Duration, missing []ContainerKey, errs []error) { + usage = make(map[ContainerKey]ContainerUsage, len(keys)) + for _, k := range keys { + prev, okPrev := w.Prev.Containers[k] + cur, okCur := w.Cur.Containers[k] + if !okPrev || !okCur { + missing = append(missing, k) + continue + } + u, err := usageBetween(prev, cur) + if err != nil { + errs = append(errs, err) + continue + } + usage[k] = u + if d := absDuration(prev.At.Sub(w.T0)); d > spread { + spread = d + } + if d := absDuration(cur.At.Sub(w.T1)); d > spread { + spread = d + } + } + return usage, spread, missing, errs +} + +func absDuration(d time.Duration) time.Duration { + if d < 0 { + return -d + } + return d +} + +// WindowDriver turns cAdvisor's housekeeping cadence into a stream of +// intervals. Its tick rate is the kubelet's, typically ~10s — the real +// resolution of any container CPU number on a kubelet-managed node. +type WindowDriver struct { + Client Scraper + // Anchor is the container whose timestamp defines the tick. It should be + // the one whose CPU matters most: everything else is then aligned to the + // series the run is actually about. + Anchor ContainerKey + // PollInterval is how often to re-fetch while waiting for the anchor + // timestamp to move. Well below the kubelet cadence, so the observed + // interval boundaries are the kubelet's and not an artifact of polling. + PollInterval time.Duration + // MaxWait bounds one Next call. Exceeding it means the kubelet stopped + // housekeeping, which is a broken rig rather than a slow one. + MaxWait time.Duration + + prev CadvisorScrape + prevAt time.Time + primed bool +} + +// Prime takes the first scrape, establishing T0 for the first window. Called +// once per arm, after the router pod is up and before load starts. +func (d *WindowDriver) Prime(ctx context.Context) error { + s, err := d.Client.Scrape(ctx) + if err != nil { + return err + } + anchor, ok := s.Containers[d.Anchor] + if !ok { + return fmt.Errorf("%w: %s", ErrAnchorMissing, d.Anchor) + } + if anchor.At.IsZero() { + return fmt.Errorf("%s: cadvisor exposed no sample timestamp; window alignment is impossible without it", d.Anchor) + } + d.prev, d.prevAt, d.primed = s, anchor.At, true + return nil +} + +// Skew reports how far the anchor's most recent cAdvisor sample lags the local +// clock. It conflates real clock skew with housekeeping age, so it is an upper +// bound rather than a correction. +func (d *WindowDriver) Skew() (time.Duration, bool) { + if !d.primed { + return 0, false + } + return d.prev.SkewAgainst(d.Anchor) +} + +// Next blocks until the anchor's cAdvisor timestamp advances and returns the +// interval between the previous timestamp and the new one. +func (d *WindowDriver) Next(ctx context.Context) (Window, error) { + if !d.primed { + if err := d.Prime(ctx); err != nil { + return Window{}, err + } + } + poll := d.PollInterval + if poll <= 0 { + poll = time.Second + } + maxWait := d.MaxWait + if maxWait <= 0 { + maxWait = 2 * time.Minute + } + deadline := time.Now().Add(maxWait) + + for polls := 1; ; polls++ { + s, err := d.Client.Scrape(ctx) + if err != nil { + return Window{}, err + } + anchor, ok := s.Containers[d.Anchor] + if !ok { + return Window{}, fmt.Errorf("%w: %s", ErrAnchorMissing, d.Anchor) + } + if anchor.At.After(d.prevAt) { + w := Window{T0: d.prevAt, T1: anchor.At, Prev: d.prev, Cur: s, Polls: polls} + d.prev, d.prevAt = s, anchor.At + return w, nil + } + // The kubelet has not housekept since the last window; emitting a + // record here would pair fresh load numbers with a stale CPU reading. + if time.Now().After(deadline) { + return Window{}, fmt.Errorf("%s: cadvisor timestamp stuck at %s for %v across %d polls", + d.Anchor, d.prevAt, maxWait, polls) + } + select { + case <-ctx.Done(): + return Window{}, ctx.Err() + case <-time.After(poll): + } + } +}