Group related findings and link topic details

README.md

@@ -713,6 +713,14 @@ and the class name rather than recorded, so a new suite is linked the day it lan
 is that moving a module without updating `SOURCE_ROOTS` in `site/assets/status.js` gives a
 404 rather than a missing link.
 
+Failures also link into the relevant topic page, down to the exact suite result where that page
+has suite-level evidence. Repeated failures are folded into likely-related groups using a weighted
+score: test class 25%, test method 15%, exception type 15%, message-token similarity 20%, and
+stack-frame similarity 25%. Every result in a group must score at least 65% against every other
+member, which prevents a chain of weak matches from swallowing unrelated failures. The page shows
+the score range and the per-result evidence; the pure scorer and its tests live in
+`site/assets/finding-groups.js` and `site/assets/finding-groups.test.js`.
+
 Three distinctions the page depends on, all decided when the results are collected:
 
 - The Gradle task a suite ran under decides whether a failure is **failing** or a

site/assets/finding-groups.js

@@ -0,0 +1,121 @@
+/* Pure helpers for relating findings. Kept separate from status.js so the scoring can be
+ * exercised with Node without inventing a browser test harness. */
+(function (root) {
+  "use strict";
+
+  const WEIGHTS = { testClass: 25, testMethod: 15, exception: 15, message: 20, stacktrace: 25 };
+  const DEFAULT_THRESHOLD = 65;
+
+  const baseSuiteName = (suite) => (suite.name || "").split(" · ")[0];
+
+  function words(value) {
+    return new Set((value || "")
+      .replace(/([a-z])([A-Z])/g, "$1 $2")
+      .toLowerCase()
+      .replace(/0x[0-9a-f]+/g, " <hex> ")
+      .replace(/\b\d+(?:\.\d+)*\b/g, " <n> ")
+      .match(/[a-z_$<>]+/g) || []);
+  }
+
+  function jaccard(left, right) {
+    if (!left.size && !right.size) return 0;
+    let intersection = 0;
+    for (const value of left) if (right.has(value)) intersection += 1;
+    return intersection / (left.size + right.size - intersection);
+  }
+
+  function exceptionName(item) {
+    const text = `${item.testCase.message || ""}\n${item.testCase.detail || ""}`;
+    const match = text.match(/(?:^|\n|Caused by:\s+|Suppressed:\s+)([\w.$]+(?:Exception|Error))(?::|\s|$)/);
+    return match ? match[1] : "";
+  }
+
+  function stackFrames(item) {
+    const frames = new Set();
+    for (const line of (item.testCase.detail || "").split("\n")) {
+      const match = line.match(/^\s*at\s+([^\s(]+)/);
+      if (match) frames.add(match[1].replace(/\$\d+/g, () => "$<n>"));
+    }
+    return frames;
+  }
+
+  function testClass(item) {
+    return item.testCase.className || item.suite.className || baseSuiteName(item.suite);
+  }
+
+  function similarity(left, right) {
+    const leftException = exceptionName(left);
+    const rightException = exceptionName(right);
+    const components = {
+      testClass: testClass(left) && testClass(left) === testClass(right) ? 1 : 0,
+      testMethod: jaccard(words(left.testCase.name), words(right.testCase.name)),
+      exception: leftException && leftException === rightException ? 1 : 0,
+      message: jaccard(words(left.testCase.message), words(right.testCase.message)),
+      stacktrace: jaccard(stackFrames(left), stackFrames(right)),
+    };
+    const score = Math.round(Object.entries(WEIGHTS)
+      .reduce((total, [name, weight]) => total + components[name] * weight, 0));
+    return { score, components };
+  }
+
+  /* Complete-link clustering: a new result must clear the threshold against every member.
+   * That deliberately avoids a chain of weak similarities turning into one giant incident. */
+  function cluster(items, threshold = DEFAULT_THRESHOLD) {
+    const groups = [];
+    for (const item of items) {
+      let best = null;
+      for (const group of groups) {
+        const matches = group.items.map((member) => similarity(item, member).score);
+        const minimum = Math.min(...matches);
+        const average = matches.reduce((sum, score) => sum + score, 0) / matches.length;
+        if (minimum >= threshold && (!best || average > best.average)) {
+          best = { group, average };
+        }
+      }
+      if (best) best.group.items.push(item);
+      else groups.push({ items: [item] });
+    }
+
+    for (const group of groups) {
+      group.representative = group.items
+        .map((item) => ({
+          item,
+          average: group.items.reduce((sum, other) => sum + similarity(item, other).score, 0) / group.items.length,
+        }))
+        .sort((a, b) => b.average - a.average)[0].item;
+      group.matches = new Map(group.items.map((item) => [item, similarity(group.representative, item)]));
+    }
+    return groups;
+  }
+
+  const ECH_SUITES = /^(?:EncryptedClientHello|PublicEncryptedClientHello|Ech(?:Conscrypt|ClientHello|Grease)?|ClientHelloExtensions)Test$/i;
+  const TOPICS = [
+    [ECH_SUITES, "ech", "Encrypted Client Hello", "all-ech-results"],
+    [/(?:Dns|Doh|HttpsRecord|HappyEyeballs|SvcParam)/i, "dns", "DNS", "current-results"],
+    [/(?:Loom)/i, "loom", "Virtual threads", "current-results"],
+    [/(?:Proxy)/i, "proxies", "Proxies", "current-results"],
+    [/(?:Tls|ConnectionSpec|Certificate|Pinning|BadChain|ClientHello|Sni|Alpn|LetsEncrypt|MockServer)/i, "tls", "TLS", "current-results"],
+    [/(?:GoHttpbin|HttpSemantics|Hostile|Http2|AltSvc|TestServer)/i, "test-servers", "Test servers", "current-results"],
+  ];
+
+  function topicFor(item) {
+    const name = baseSuiteName(item.suite);
+    const match = TOPICS.find(([pattern]) => pattern.test(name));
+    if (!match) return null;
+    const [, slug, label, defaultAnchor] = match;
+    const anchor = slug === "ech"
+      ? defaultAnchor
+      : `result-${`${item.suite.name}-${item.version.okhttpVersion}`.toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}`;
+    return { label, href: `topics/${slug}.html#${anchor}` };
+  }
+
+  root.FindingGroups = {
+    DEFAULT_THRESHOLD,
+    WEIGHTS,
+    cluster,
+    exceptionName,
+    similarity,
+    topicFor,
+  };
+})(typeof globalThis === "undefined" ? window : globalThis);

site/assets/finding-groups.test.js

@@ -0,0 +1,49 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+require("./finding-groups.js");
+const { cluster, similarity, topicFor } = globalThis.FindingGroups;
+
+function result({ suite = "PublicEncryptedClientHelloTest", method, message, detail, className }) {
+  return {
+    kind: "failed",
+    version: { okhttpVersion: "5.5.0-SNAPSHOT" },
+    suite: { name: `${suite} · API 37`, className: className || `example.${suite}` },
+    testCase: { name: method, message, detail, className: className || `example.${suite}` },
+  };
+}
+
+const networkTrace = (method, line = 285) => `java.net.ConnectException: Failed to connect to /1.1.1.1:443
+  at okhttp3.internal.connection.ConnectPlan.connectSocket(ConnectPlan.kt:${line})
+  at okhttp3.internal.connection.ConnectPlan.connectTcp(ConnectPlan.kt:144)
+  at java.lang.Thread.run(Thread.java:1572)`;
+
+test("groups different methods in one class when the exception and trace agree", () => {
+  const a = result({ method: "cloudflareUsesEch", message: "java.net.ConnectException: Failed to connect to /1.1.1.1:443", detail: networkTrace("cloudflareUsesEch") });
+  const b = result({ method: "echIsAcceptedOnDefoIe", message: "java.net.ConnectException: Failed to connect to /1.1.1.1:443", detail: networkTrace("echIsAcceptedOnDefoIe", 291) });
+  assert.ok(similarity(a, b).score >= 65);
+  assert.equal(cluster([a, b]).length, 1);
+});
+
+test("keeps unrelated assertions separate even when their class matches", () => {
+  const a = result({ suite: "DnsFailureTest", method: "returnsUnknownHost", message: "expected UnknownHostException but was IOException", detail: "org.opentest4j.AssertionFailedError: expected UnknownHostException\n  at example.DnsFailureTest.returnsUnknownHost(DnsFailureTest.kt:40)" });
+  const b = result({ suite: "DnsFailureTest", method: "preservesCause", message: "expected cause to be present", detail: "org.opentest4j.AssertionFailedError: expected cause\n  at example.DnsFailureTest.preservesCause(DnsFailureTest.kt:90)" });
+  assert.ok(similarity(a, b).score < 65);
+  assert.equal(cluster([a, b]).length, 2);
+});
+
+test("links a result to its suite on the relevant secondary page", () => {
+  const dns = result({ suite: "DnsFailureTest", method: "returnsUnknownHost", message: "failure", detail: "" });
+  assert.deepEqual(topicFor(dns), {
+    label: "DNS",
+    href: "topics/dns.html#result-dnsfailuretest-api-37-5-5-0-snapshot",
+  });
+});
+
+test("links ECH findings to the complete ECH evidence matrix", () => {
+  const ech = result({ method: "cloudflareUsesEch", message: "failure", detail: "" });
+  assert.deepEqual(topicFor(ech), {
+    label: "Encrypted Client Hello",
+    href: "topics/ech.html#all-ech-results",
+  });
+});

site/assets/status.js

@@ -205,54 +205,102 @@ function renderFailures(snapshot) {
           : suite.reporting && suite.severity !== "critical" ? "finding"
           : "failed";
 
-        const source = sourceUrl(suite);
-        // Open, because the assertion is the reason to come back to this page — a triangle to
-        // click before you can read it is one more step for the thing you came for. Skips are
-        // the exception: the reason is one line, and the Endpoints table already carries it.
-        // Open, because the assertion is the reason to come back to this page. Two exceptions,
-        // both cases where the one line that matters is already in the summary: a skip, and a
-        // failure that was predicted — those are folded so the unpredicted ones stand out.
-        const detail = el("details", {
-          className: "finding-detail",
-          open: kind !== "skipped" && kind !== "expected",
-        }, [
-          el("summary", {}, [
-            pill(kind),
-            ` ${suite.name}.${testCase.name} — ${version.okhttpVersion}`,
-          ]),
-          // The trace usually opens with the message verbatim, so printing both repeats the
-          // one line that matters. Show the message alone only when it isn't already there.
-          testCase.expectedReason
-            ? el("p", { className: "expected-reason", textContent: testCase.expectedReason })
-            : null,
-          el("pre", {
-            textContent:
-              (testCase.detail?.startsWith(testCase.message)
-                ? testCase.detail
-                : [testCase.message, testCase.detail].filter(Boolean).join("\n\n")) ||
-              "No detail recorded.",
-          }),
-          source
-            ? el("p", { className: "card-label" }, [
-                el("a", { href: source, textContent: `${suite.className} ↗` }),
-              ])
-            : null,
-        ]);
-        detail.dataset.kind = kind;
-        items.push({ kind, detail });
+        items.push({ kind, version, suite, testCase });
       }
     }
   }
 
   items.sort((a, b) => rank[a.kind] - rank[b.kind]);
 
+  function findingDetail(item, match = null, open = false) {
+    const { kind, version, suite, testCase } = item;
+    const source = sourceUrl(suite);
+    const topic = FindingGroups.topicFor(item);
+    const why = match && match.score < 100
+      ? [
+          `${match.score}% match`,
+          match.components.testClass === 1 ? "same class" : null,
+          match.components.testMethod === 1 ? "same method" : null,
+          match.components.exception === 1 ? "same exception" : null,
+          match.components.message ? `${Math.round(match.components.message * 100)}% message` : null,
+          match.components.stacktrace ? `${Math.round(match.components.stacktrace * 100)}% stack` : null,
+        ].filter(Boolean).join(" · ")
+      : "";
+    const detail = el("details", { className: "finding-detail", open }, [
+      el("summary", {}, [
+        pill(kind),
+        ` ${suite.name}.${testCase.name} — ${version.okhttpVersion}`,
+      ]),
+      why ? el("p", { className: "similarity-note", textContent: why }) : null,
+      testCase.expectedReason
+        ? el("p", { className: "expected-reason", textContent: testCase.expectedReason })
+        : null,
+      el("pre", {
+        textContent:
+          (testCase.detail?.startsWith(testCase.message)
+            ? testCase.detail
+            : [testCase.message, testCase.detail].filter(Boolean).join("\n\n")) ||
+          "No detail recorded.",
+      }),
+      source || topic
+        ? el("p", { className: "finding-links card-label" }, [
+            topic ? el("a", { href: topic.href, textContent: `${topic.label} details →` }) : null,
+            source ? el("a", { href: source, textContent: `${suite.className} ↗` }) : null,
+          ])
+        : null,
+    ]);
+    detail.dataset.kind = kind;
+    return detail;
+  }
+
+  const groups = FindingGroups.cluster(items).sort((a, b) =>
+    Math.min(...a.items.map((item) => rank[item.kind])) -
+    Math.min(...b.items.map((item) => rank[item.kind])),
+  );
+
+  const rendered = groups.map((group) => {
+    if (group.items.length === 1) {
+      const item = group.items[0];
+      return findingDetail(item, null, item.kind !== "skipped" && item.kind !== "expected");
+    }
+
+    const representative = group.representative;
+    const strongestKind = group.items.map((item) => item.kind)
+      .sort((a, b) => rank[a] - rank[b])[0];
+    const scores = group.items
+      .filter((item) => item !== representative)
+      .map((item) => group.matches.get(item).score);
+    const message = (representative.testCase.message || FindingGroups.exceptionName(representative) ||
+      `${representative.suite.name}.${representative.testCase.name}`).split("\n")[0];
+    const shortMessage = message.length > 135 ? `${message.slice(0, 132)}…` : message;
+    const range = scores.length
+      ? `${Math.min(...scores)}${Math.min(...scores) === Math.max(...scores) ? "" : `–${Math.max(...scores)}`}% match`
+      : "";
+    const topic = FindingGroups.topicFor(representative);
+
+    const incident = el("details", { className: "incident-group" }, [
+      el("summary", {}, [
+        pill(strongestKind),
+        el("strong", { textContent: `${plural(group.items.length, "result")} likely related` }),
+        el("span", { className: "incident-message", textContent: shortMessage }),
+      ]),
+      el("div", { className: "incident-meta" }, [
+        el("span", { textContent: `${range} · weighted by test class, method, exception, message and stack trace` }),
+        topic ? el("a", { href: topic.href, textContent: `${topic.label} details →` }) : null,
+      ]),
+      ...group.items.map((item) => findingDetail(item, group.matches.get(item))),
+    ]);
+    incident.dataset.kind = strongestKind;
+    return incident;
+  });
+
   const body = document.getElementById("failure-list");
   if (!items.length) {
     body.replaceChildren(
       el("p", { textContent: "Everything ran, and everything passed." }),
     );
   } else {
-    body.replaceChildren(...items.map((item) => item.detail));
+    body.replaceChildren(...rendered);
   }
 }
 

site/assets/style.css

@@ -312,6 +312,45 @@ details.finding-detail pre {
   line-height: 1.5;
 }
 
+.finding-links, .incident-meta {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.35rem 1rem;
+}
+
+.similarity-note {
+  color: var(--text-muted);
+  font-size: 0.78rem;
+  margin: 0.55rem 0 0;
+}
+
+details.incident-group {
+  border: 1px solid var(--border);
+  border-left: 4px solid var(--fail);
+  border-radius: var(--radius);
+  background: var(--bg-sunken);
+  padding: 0.75rem 0.9rem;
+  margin: 0.75rem 0;
+}
+
+details.incident-group[data-kind="finding"],
+details.incident-group[data-kind="expected"] { border-left-color: var(--finding); }
+details.incident-group[data-kind="skipped"] { border-left-color: var(--skip); }
+
+details.incident-group > summary {
+  cursor: pointer;
+  display: flex;
+  align-items: baseline;
+  flex-wrap: wrap;
+  gap: 0.4rem 0.65rem;
+  font-family: var(--mono);
+  font-size: 0.88rem;
+}
+
+.incident-message { color: var(--text-muted); font-size: 0.8rem; }
+.incident-meta { color: var(--text-muted); font-size: 0.78rem; margin: 0.7rem 0; }
+details.incident-group > details.finding-detail { margin-left: 0.25rem; background: var(--bg-raised); }
+
 /* ---- open work ---- */
 
 .issue-groups {

site/index.html

@@ -187,6 +187,7 @@ <h2>About these results</h2>
   </div>
 </footer>
 
+<script src="assets/finding-groups.js"></script>
 <script src="assets/status.js"></script>
 </body>
 </html>

site/topics/ech.html

@@ -152,7 +152,7 @@ <h2>The GREASE case</h2>
     failed test.
   </p>
 
-  <h2>All ECH test cases</h2>
+  <h2 id="all-ech-results">All ECH test cases</h2>
   <p>
     This is the complete result set, including fixture and public-server connections, GREASE
     compatibility, DNS hand-off through Conscrypt, and the local ClientHello byte and extension