diff --git a/README.md b/README.md index 62a918f..98e0ca2 100644 --- a/README.md +++ b/README.md @@ -713,6 +713,14 @@ and the class name rather than recorded, so a new suite is linked the day it lan is that moving a module without updating `SOURCE_ROOTS` in `site/assets/status.js` gives a 404 rather than a missing link. +Failures also link into the relevant topic page, down to the exact suite result where that page +has suite-level evidence. Repeated failures are folded into likely-related groups using a weighted +score: test class 25%, test method 15%, exception type 15%, message-token similarity 20%, and +stack-frame similarity 25%. Every result in a group must score at least 65% against every other +member, which prevents a chain of weak matches from swallowing unrelated failures. The page shows +the score range and the per-result evidence; the pure scorer and its tests live in +`site/assets/finding-groups.js` and `site/assets/finding-groups.test.js`. + Three distinctions the page depends on, all decided when the results are collected: - The Gradle task a suite ran under decides whether a failure is **failing** or a diff --git a/site/assets/finding-groups.js b/site/assets/finding-groups.js new file mode 100644 index 0000000..e7b7d91 --- /dev/null +++ b/site/assets/finding-groups.js @@ -0,0 +1,121 @@ +/* Pure helpers for relating findings. Kept separate from status.js so the scoring can be + * exercised with Node without inventing a browser test harness. */ +(function (root) { + "use strict"; + + const WEIGHTS = { testClass: 25, testMethod: 15, exception: 15, message: 20, stacktrace: 25 }; + const DEFAULT_THRESHOLD = 65; + + const baseSuiteName = (suite) => (suite.name || "").split(" · ")[0]; + + function words(value) { + return new Set((value || "") + .replace(/([a-z])([A-Z])/g, "$1 $2") + .toLowerCase() + .replace(/0x[0-9a-f]+/g, " ") + .replace(/\b\d+(?:\.\d+)*\b/g, " ") + .match(/[a-z_$<>]+/g) || []); + } + + function jaccard(left, right) { + if (!left.size && !right.size) return 0; + let intersection = 0; + for (const value of left) if (right.has(value)) intersection += 1; + return intersection / (left.size + right.size - intersection); + } + + function exceptionName(item) { + const text = `${item.testCase.message || ""}\n${item.testCase.detail || ""}`; + const match = text.match(/(?:^|\n|Caused by:\s+|Suppressed:\s+)([\w.$]+(?:Exception|Error))(?::|\s|$)/); + return match ? match[1] : ""; + } + + function stackFrames(item) { + const frames = new Set(); + for (const line of (item.testCase.detail || "").split("\n")) { + const match = line.match(/^\s*at\s+([^\s(]+)/); + if (match) frames.add(match[1].replace(/\$\d+/g, () => "$")); + } + return frames; + } + + function testClass(item) { + return item.testCase.className || item.suite.className || baseSuiteName(item.suite); + } + + function similarity(left, right) { + const leftException = exceptionName(left); + const rightException = exceptionName(right); + const components = { + testClass: testClass(left) && testClass(left) === testClass(right) ? 1 : 0, + testMethod: jaccard(words(left.testCase.name), words(right.testCase.name)), + exception: leftException && leftException === rightException ? 1 : 0, + message: jaccard(words(left.testCase.message), words(right.testCase.message)), + stacktrace: jaccard(stackFrames(left), stackFrames(right)), + }; + const score = Math.round(Object.entries(WEIGHTS) + .reduce((total, [name, weight]) => total + components[name] * weight, 0)); + return { score, components }; + } + + /* Complete-link clustering: a new result must clear the threshold against every member. + * That deliberately avoids a chain of weak similarities turning into one giant incident. */ + function cluster(items, threshold = DEFAULT_THRESHOLD) { + const groups = []; + for (const item of items) { + let best = null; + for (const group of groups) { + const matches = group.items.map((member) => similarity(item, member).score); + const minimum = Math.min(...matches); + const average = matches.reduce((sum, score) => sum + score, 0) / matches.length; + if (minimum >= threshold && (!best || average > best.average)) { + best = { group, average }; + } + } + if (best) best.group.items.push(item); + else groups.push({ items: [item] }); + } + + for (const group of groups) { + group.representative = group.items + .map((item) => ({ + item, + average: group.items.reduce((sum, other) => sum + similarity(item, other).score, 0) / group.items.length, + })) + .sort((a, b) => b.average - a.average)[0].item; + group.matches = new Map(group.items.map((item) => [item, similarity(group.representative, item)])); + } + return groups; + } + + const ECH_SUITES = /^(?:EncryptedClientHello|PublicEncryptedClientHello|Ech(?:Conscrypt|ClientHello|Grease)?|ClientHelloExtensions)Test$/i; + const TOPICS = [ + [ECH_SUITES, "ech", "Encrypted Client Hello", "all-ech-results"], + [/(?:Dns|Doh|HttpsRecord|HappyEyeballs|SvcParam)/i, "dns", "DNS", "current-results"], + [/(?:Loom)/i, "loom", "Virtual threads", "current-results"], + [/(?:Proxy)/i, "proxies", "Proxies", "current-results"], + [/(?:Tls|ConnectionSpec|Certificate|Pinning|BadChain|ClientHello|Sni|Alpn|LetsEncrypt|MockServer)/i, "tls", "TLS", "current-results"], + [/(?:GoHttpbin|HttpSemantics|Hostile|Http2|AltSvc|TestServer)/i, "test-servers", "Test servers", "current-results"], + ]; + + function topicFor(item) { + const name = baseSuiteName(item.suite); + const match = TOPICS.find(([pattern]) => pattern.test(name)); + if (!match) return null; + const [, slug, label, defaultAnchor] = match; + const anchor = slug === "ech" + ? defaultAnchor + : `result-${`${item.suite.name}-${item.version.okhttpVersion}`.toLowerCase() + .replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}`; + return { label, href: `topics/${slug}.html#${anchor}` }; + } + + root.FindingGroups = { + DEFAULT_THRESHOLD, + WEIGHTS, + cluster, + exceptionName, + similarity, + topicFor, + }; +})(typeof globalThis === "undefined" ? window : globalThis); diff --git a/site/assets/finding-groups.test.js b/site/assets/finding-groups.test.js new file mode 100644 index 0000000..28ebbf7 --- /dev/null +++ b/site/assets/finding-groups.test.js @@ -0,0 +1,49 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +require("./finding-groups.js"); +const { cluster, similarity, topicFor } = globalThis.FindingGroups; + +function result({ suite = "PublicEncryptedClientHelloTest", method, message, detail, className }) { + return { + kind: "failed", + version: { okhttpVersion: "5.5.0-SNAPSHOT" }, + suite: { name: `${suite} · API 37`, className: className || `example.${suite}` }, + testCase: { name: method, message, detail, className: className || `example.${suite}` }, + }; +} + +const networkTrace = (method, line = 285) => `java.net.ConnectException: Failed to connect to /1.1.1.1:443 + at okhttp3.internal.connection.ConnectPlan.connectSocket(ConnectPlan.kt:${line}) + at okhttp3.internal.connection.ConnectPlan.connectTcp(ConnectPlan.kt:144) + at java.lang.Thread.run(Thread.java:1572)`; + +test("groups different methods in one class when the exception and trace agree", () => { + const a = result({ method: "cloudflareUsesEch", message: "java.net.ConnectException: Failed to connect to /1.1.1.1:443", detail: networkTrace("cloudflareUsesEch") }); + const b = result({ method: "echIsAcceptedOnDefoIe", message: "java.net.ConnectException: Failed to connect to /1.1.1.1:443", detail: networkTrace("echIsAcceptedOnDefoIe", 291) }); + assert.ok(similarity(a, b).score >= 65); + assert.equal(cluster([a, b]).length, 1); +}); + +test("keeps unrelated assertions separate even when their class matches", () => { + const a = result({ suite: "DnsFailureTest", method: "returnsUnknownHost", message: "expected UnknownHostException but was IOException", detail: "org.opentest4j.AssertionFailedError: expected UnknownHostException\n at example.DnsFailureTest.returnsUnknownHost(DnsFailureTest.kt:40)" }); + const b = result({ suite: "DnsFailureTest", method: "preservesCause", message: "expected cause to be present", detail: "org.opentest4j.AssertionFailedError: expected cause\n at example.DnsFailureTest.preservesCause(DnsFailureTest.kt:90)" }); + assert.ok(similarity(a, b).score < 65); + assert.equal(cluster([a, b]).length, 2); +}); + +test("links a result to its suite on the relevant secondary page", () => { + const dns = result({ suite: "DnsFailureTest", method: "returnsUnknownHost", message: "failure", detail: "" }); + assert.deepEqual(topicFor(dns), { + label: "DNS", + href: "topics/dns.html#result-dnsfailuretest-api-37-5-5-0-snapshot", + }); +}); + +test("links ECH findings to the complete ECH evidence matrix", () => { + const ech = result({ method: "cloudflareUsesEch", message: "failure", detail: "" }); + assert.deepEqual(topicFor(ech), { + label: "Encrypted Client Hello", + href: "topics/ech.html#all-ech-results", + }); +}); diff --git a/site/assets/status.js b/site/assets/status.js index 0697761..fe07565 100644 --- a/site/assets/status.js +++ b/site/assets/status.js @@ -205,54 +205,102 @@ function renderFailures(snapshot) { : suite.reporting && suite.severity !== "critical" ? "finding" : "failed"; - const source = sourceUrl(suite); - // Open, because the assertion is the reason to come back to this page — a triangle to - // click before you can read it is one more step for the thing you came for. Skips are - // the exception: the reason is one line, and the Endpoints table already carries it. - // Open, because the assertion is the reason to come back to this page. Two exceptions, - // both cases where the one line that matters is already in the summary: a skip, and a - // failure that was predicted — those are folded so the unpredicted ones stand out. - const detail = el("details", { - className: "finding-detail", - open: kind !== "skipped" && kind !== "expected", - }, [ - el("summary", {}, [ - pill(kind), - ` ${suite.name}.${testCase.name} — ${version.okhttpVersion}`, - ]), - // The trace usually opens with the message verbatim, so printing both repeats the - // one line that matters. Show the message alone only when it isn't already there. - testCase.expectedReason - ? el("p", { className: "expected-reason", textContent: testCase.expectedReason }) - : null, - el("pre", { - textContent: - (testCase.detail?.startsWith(testCase.message) - ? testCase.detail - : [testCase.message, testCase.detail].filter(Boolean).join("\n\n")) || - "No detail recorded.", - }), - source - ? el("p", { className: "card-label" }, [ - el("a", { href: source, textContent: `${suite.className} ↗` }), - ]) - : null, - ]); - detail.dataset.kind = kind; - items.push({ kind, detail }); + items.push({ kind, version, suite, testCase }); } } } items.sort((a, b) => rank[a.kind] - rank[b.kind]); + function findingDetail(item, match = null, open = false) { + const { kind, version, suite, testCase } = item; + const source = sourceUrl(suite); + const topic = FindingGroups.topicFor(item); + const why = match && match.score < 100 + ? [ + `${match.score}% match`, + match.components.testClass === 1 ? "same class" : null, + match.components.testMethod === 1 ? "same method" : null, + match.components.exception === 1 ? "same exception" : null, + match.components.message ? `${Math.round(match.components.message * 100)}% message` : null, + match.components.stacktrace ? `${Math.round(match.components.stacktrace * 100)}% stack` : null, + ].filter(Boolean).join(" · ") + : ""; + const detail = el("details", { className: "finding-detail", open }, [ + el("summary", {}, [ + pill(kind), + ` ${suite.name}.${testCase.name} — ${version.okhttpVersion}`, + ]), + why ? el("p", { className: "similarity-note", textContent: why }) : null, + testCase.expectedReason + ? el("p", { className: "expected-reason", textContent: testCase.expectedReason }) + : null, + el("pre", { + textContent: + (testCase.detail?.startsWith(testCase.message) + ? testCase.detail + : [testCase.message, testCase.detail].filter(Boolean).join("\n\n")) || + "No detail recorded.", + }), + source || topic + ? el("p", { className: "finding-links card-label" }, [ + topic ? el("a", { href: topic.href, textContent: `${topic.label} details →` }) : null, + source ? el("a", { href: source, textContent: `${suite.className} ↗` }) : null, + ]) + : null, + ]); + detail.dataset.kind = kind; + return detail; + } + + const groups = FindingGroups.cluster(items).sort((a, b) => + Math.min(...a.items.map((item) => rank[item.kind])) - + Math.min(...b.items.map((item) => rank[item.kind])), + ); + + const rendered = groups.map((group) => { + if (group.items.length === 1) { + const item = group.items[0]; + return findingDetail(item, null, item.kind !== "skipped" && item.kind !== "expected"); + } + + const representative = group.representative; + const strongestKind = group.items.map((item) => item.kind) + .sort((a, b) => rank[a] - rank[b])[0]; + const scores = group.items + .filter((item) => item !== representative) + .map((item) => group.matches.get(item).score); + const message = (representative.testCase.message || FindingGroups.exceptionName(representative) || + `${representative.suite.name}.${representative.testCase.name}`).split("\n")[0]; + const shortMessage = message.length > 135 ? `${message.slice(0, 132)}…` : message; + const range = scores.length + ? `${Math.min(...scores)}${Math.min(...scores) === Math.max(...scores) ? "" : `–${Math.max(...scores)}`}% match` + : ""; + const topic = FindingGroups.topicFor(representative); + + const incident = el("details", { className: "incident-group" }, [ + el("summary", {}, [ + pill(strongestKind), + el("strong", { textContent: `${plural(group.items.length, "result")} likely related` }), + el("span", { className: "incident-message", textContent: shortMessage }), + ]), + el("div", { className: "incident-meta" }, [ + el("span", { textContent: `${range} · weighted by test class, method, exception, message and stack trace` }), + topic ? el("a", { href: topic.href, textContent: `${topic.label} details →` }) : null, + ]), + ...group.items.map((item) => findingDetail(item, group.matches.get(item))), + ]); + incident.dataset.kind = strongestKind; + return incident; + }); + const body = document.getElementById("failure-list"); if (!items.length) { body.replaceChildren( el("p", { textContent: "Everything ran, and everything passed." }), ); } else { - body.replaceChildren(...items.map((item) => item.detail)); + body.replaceChildren(...rendered); } } diff --git a/site/assets/style.css b/site/assets/style.css index fde0cd1..6755659 100644 --- a/site/assets/style.css +++ b/site/assets/style.css @@ -312,6 +312,45 @@ details.finding-detail pre { line-height: 1.5; } +.finding-links, .incident-meta { + display: flex; + flex-wrap: wrap; + gap: 0.35rem 1rem; +} + +.similarity-note { + color: var(--text-muted); + font-size: 0.78rem; + margin: 0.55rem 0 0; +} + +details.incident-group { + border: 1px solid var(--border); + border-left: 4px solid var(--fail); + border-radius: var(--radius); + background: var(--bg-sunken); + padding: 0.75rem 0.9rem; + margin: 0.75rem 0; +} + +details.incident-group[data-kind="finding"], +details.incident-group[data-kind="expected"] { border-left-color: var(--finding); } +details.incident-group[data-kind="skipped"] { border-left-color: var(--skip); } + +details.incident-group > summary { + cursor: pointer; + display: flex; + align-items: baseline; + flex-wrap: wrap; + gap: 0.4rem 0.65rem; + font-family: var(--mono); + font-size: 0.88rem; +} + +.incident-message { color: var(--text-muted); font-size: 0.8rem; } +.incident-meta { color: var(--text-muted); font-size: 0.78rem; margin: 0.7rem 0; } +details.incident-group > details.finding-detail { margin-left: 0.25rem; background: var(--bg-raised); } + /* ---- open work ---- */ .issue-groups { diff --git a/site/index.html b/site/index.html index 132bc16..6deb532 100644 --- a/site/index.html +++ b/site/index.html @@ -187,6 +187,7 @@

About these results

+ diff --git a/site/topics/ech.html b/site/topics/ech.html index 0b3677b..88f8d40 100644 --- a/site/topics/ech.html +++ b/site/topics/ech.html @@ -152,7 +152,7 @@

The GREASE case

failed test.

-

All ECH test cases

+

All ECH test cases

This is the complete result set, including fixture and public-server connections, GREASE compatibility, DNS hand-off through Conscrypt, and the local ClientHello byte and extension