diff --git a/.github/workflows/cgo.yml b/.github/workflows/cgo.yml index b983c451c65..785985d5c41 100644 --- a/.github/workflows/cgo.yml +++ b/.github/workflows/cgo.yml @@ -50,6 +50,8 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: Verify CGO/CJS workflow purity + run: bash scripts/check-cgo-cjs-workflow-purity.sh - name: Cache repository checkout uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -2508,216 +2510,6 @@ jobs: path: conformance-output.txt retention-days: 7 - notify-failure: - name: Notify on CGO Failure - runs-on: ubuntu-latest - timeout-minutes: 5 - if: always() && github.ref == 'refs/heads/main' - needs: - - checkout-cache - - verify-integration-build - - test - - canary-go - - build - - build-wasm - - bench - - check-validator-sizes - - lint-go - - lint-error-messages - - actions-build - - fuzz - - security - - security-scan - - mcp-server-compile-test - - cross-platform-build - - alpine-container-test - - safe-outputs-conformance - permissions: - issues: write - steps: - - name: Check for job failures and create issue - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - const needs = ${{ toJSON(needs) }}; - const failedJobs = Object.entries(needs) - .filter(([, job]) => job.result === 'failure') - .map(([name]) => name); - const hasFuzzFailure = failedJobs.some(name => name === 'fuzz' || name.startsWith('fuzz-')); - - if (failedJobs.length === 0) { - core.info('No jobs failed. Nothing to do.'); - return; - } - - core.info(`Failed jobs: ${failedJobs.join(', ')}`); - - // Check for an existing open CGO failure issue to avoid duplicates - const existingIssues = await github.rest.issues.listForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - labels: 'cgo-failure', - state: 'open', - }); - - if (existingIssues.data.length > 0 && !hasFuzzFailure) { - core.info(`Existing CGO failure issue #${existingIssues.data[0].number} is still open. Skipping.`); - return; - } - - if (hasFuzzFailure) { - const existingFuzzIssues = await github.rest.issues.listForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - labels: 'cgo-fuzz-failure', - state: 'open', - }); - - const existingFuzzIssueForRun = existingFuzzIssues.data.find(issue => - issue.title.includes(`Run #${context.runNumber}`), - ); - if (existingFuzzIssueForRun) { - core.info(`Fuzz failure issue #${existingFuzzIssueForRun.number} already exists for run #${context.runNumber}. Skipping.`); - return; - } - } - - // Ensure required labels exist, creating them if missing - const requiredLabels = [['cookie', 'e4e669'], ['cgo-failure', 'b60205']]; - if (hasFuzzFailure) { - requiredLabels.push(['cgo-fuzz-failure', 'd93f0b']); - } - - for (const [label, color] of requiredLabels) { - try { - await github.rest.issues.getLabel({ owner: context.repo.owner, repo: context.repo.repo, name: label }); - } catch (e) { - if (e.status === 404) { - await github.rest.issues.createLabel({ owner: context.repo.owner, repo: context.repo.repo, name: label, color }); - } - } - } - - // Fetch all jobs for this run to get direct job links - const jobsResponse = await github.rest.actions.listJobsForWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100, - }); - - // Build a map from job name to job URL for failed jobs - const jobUrlMap = {}; - for (const job of jobsResponse.data.jobs) { - if (failedJobs.includes(job.name)) { - jobUrlMap[job.name] = job.html_url; - } - } - - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const expiresAt = new Date(Date.now() + 4 * 60 * 60 * 1000).toISOString(); - - const parseUTCOffsetMinutes = (rawOffset) => { - const offset = typeof rawOffset === 'string' ? rawOffset.trim() : ''; - const match = offset.match(/^([+-])(\d{2}):(\d{2})$/); - if (!match) return Number.NaN; - const [, sign, hoursText, minutesText] = match; - const hours = Number.parseInt(hoursText, 10); - const minutes = Number.parseInt(minutesText, 10); - if (hours > 14 || minutes > 59 || (hours === 14 && minutes !== 0)) { - return Number.NaN; - } - const direction = sign === '-' ? -1 : 1; - return direction * (hours * 60 + minutes); - }; - - let repoUTCOffset = ''; - try { - const awConfigFile = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: '.github/workflows/aw.json', - ref: context.sha, - }); - if (!Array.isArray(awConfigFile.data) && awConfigFile.data.type === 'file' && awConfigFile.data.content) { - const awConfigRaw = Buffer.from(awConfigFile.data.content, 'base64').toString('utf8'); - const awConfig = JSON.parse(awConfigRaw); - const rawUTC = typeof awConfig?.utc === 'string' ? awConfig.utc.trim() : ''; - if (!Number.isNaN(parseUTCOffsetMinutes(rawUTC))) { - repoUTCOffset = rawUTC; - } else if (rawUTC) { - core.warning(`Ignoring invalid utc offset in .github/workflows/aw.json: ${rawUTC}`); - } - } - } catch (error) { - core.warning(`Unable to read .github/workflows/aw.json UTC offset: ${error?.message || String(error)}`); - } - - // Format expiration line using the gh-aw-expires XML comment format - const expiresDate = new Date(expiresAt); - const repoOffsetMinutes = parseUTCOffsetMinutes(repoUTCOffset); - const hasRepoUTCOffset = !Number.isNaN(repoOffsetMinutes); - const displayDate = hasRepoUTCOffset - ? new Date(expiresDate.getTime() + repoOffsetMinutes * 60 * 1000) - : expiresDate; - const humanReadableDate = displayDate.toLocaleString('en-US', { - dateStyle: 'medium', - timeStyle: 'short', - timeZone: 'UTC', - }); - const humanReadableSuffix = hasRepoUTCOffset ? `UTC${repoUTCOffset}` : 'UTC'; - const expirationLine = `- [x] expires on ${humanReadableDate} ${humanReadableSuffix}`; - - const body = [ - `## CGO Workflow Failure`, - ``, - `Workflow run [#${context.runNumber}](${runUrl}) on the \`main\` branch completed with failed jobs.`, - ``, - `| Field | Value |`, - `| --- | --- |`, - `| Run ID | ${context.runId} |`, - `| Commit | ${context.sha} |`, - `| Expires | ${expiresAt} |`, - ``, - `## Failed Jobs`, - ``, - // Map job names to direct links; fall back to plain text if a job ID wasn't found - ...failedJobs.map(name => jobUrlMap[name] - ? `- [\`${name}\`](${jobUrlMap[name]})` - : `- \`${name}\``), - ``, - `> This issue expires at ${expiresAt}. Please investigate the failed jobs above and close once resolved.`, - `> ${expirationLine}`, - ]; - - if (hasFuzzFailure) { - body.splice( - 4, - 0, - `Detected failure in the \`fuzz\` job matrix. A dedicated fuzz failure label was applied so this run is tracked.`, - ``, - ); - } - - const issueBody = body.join('\n'); - - const issueLabels = ['cookie', 'cgo-failure']; - if (hasFuzzFailure) { - issueLabels.push('cgo-fuzz-failure'); - } - - const issue = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: hasFuzzFailure - ? `[CGO][FUZZ] Workflow failure on main - Run #${context.runNumber}` - : `[CGO] Workflow failure on main - Run #${context.runNumber}`, - body: issueBody, - labels: issueLabels, - }); - - core.info(`Created issue #${issue.data.number}: ${issue.data.html_url}`); - summarize-timing: name: Summarize workflow timing needs: @@ -2741,7 +2533,6 @@ jobs: - cross-platform-build - alpine-container-test - safe-outputs-conformance - - notify-failure if: always() runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.github/workflows/cjs.yml b/.github/workflows/cjs.yml index 86d65118390..7e64c7def53 100644 --- a/.github/workflows/cjs.yml +++ b/.github/workflows/cjs.yml @@ -8,6 +8,7 @@ on: - 'actions/setup/md/**' - 'pkg/cli/data/models.json' - 'scripts/**/*.js' + - 'scripts/check-cgo-cjs-workflow-purity.sh' - 'Makefile' - '.github/workflows/ci.yml' - '.github/workflows/cjs.yml' @@ -18,6 +19,7 @@ on: - 'actions/setup/md/**' - 'pkg/cli/data/models.json' - 'scripts/**/*.js' + - 'scripts/check-cgo-cjs-workflow-purity.sh' - 'Makefile' - '.github/workflows/ci.yml' - '.github/workflows/cjs.yml' @@ -35,6 +37,8 @@ jobs: with: fetch-depth: 0 persist-credentials: false + - name: Verify CGO/CJS workflow purity + run: bash scripts/check-cgo-cjs-workflow-purity.sh - name: Cache repository checkout uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -202,7 +206,7 @@ jobs: timeout-minutes: 10 permissions: contents: read - actions: write + actions: read concurrency: group: ci-${{ github.ref }}-artifact-integration cancel-in-progress: true diff --git a/Makefile b/Makefile index a543bb308b2..5189aa41fcb 100644 --- a/Makefile +++ b/Makefile @@ -482,12 +482,13 @@ bundle-js: @echo "✓ bundle-js tool built" @echo "To bundle a JavaScript file: ./bundle-js [output-file]" -# Run Bash script tests (check-stale-lock-files, check-workflow-drift) +# Run Bash script tests (check-stale-lock-files, check-workflow-drift, check-cgo-cjs-workflow-purity) .PHONY: test-scripts test-scripts: build @echo "Running Bash script tests..." bash scripts/check-stale-lock-files_test.sh bash scripts/check-workflow-drift_test.sh ./$(BINARY_NAME) + bash scripts/check-cgo-cjs-workflow-purity_test.sh @echo "✓ All Bash script tests passed" # Test all code (Go, JavaScript, wasm golden, and shell scripts) diff --git a/scripts/check-cgo-cjs-workflow-purity.sh b/scripts/check-cgo-cjs-workflow-purity.sh new file mode 100644 index 00000000000..1839cb03dec --- /dev/null +++ b/scripts/check-cgo-cjs-workflow-purity.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -eq 0 ]; then + set -- .github/workflows/cgo.yml .github/workflows/cjs.yml +fi + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +failed=0 +for workflow in "$@"; do + echo "Checking $workflow" + if [ ! -f "$workflow" ]; then + echo "Missing workflow file: $workflow" + failed=1 + continue + fi + + # These workflows should stay pure test workflows: allow only the built-in + # GITHUB_TOKEN. + disallowed_secrets_file="$tmp_dir/disallowed-secrets.txt" + if ! python3 - "$workflow" >"$disallowed_secrets_file" <<'PY'; then +import re +import sys + +workflow = sys.argv[1] +allowed = {"GITHUB_TOKEN"} + +with open(workflow, encoding="utf-8") as f: + content = f.read() + +expression_re = re.compile(r"\$\{\{(.*?)\}\}", re.DOTALL) +secret_re = re.compile( + r"\bsecrets\b\s*(?:" + r"\.\s*([A-Za-z_][A-Za-z0-9_]*)" + r"|\[\s*(['\"])(.*?)\2\s*\]" + r"|\[([^\]]*)\]" + r")", + re.DOTALL, +) + +for expression in expression_re.finditer(content): + expression_text = expression.group(1) + expression_line = content.count("\n", 0, expression.start()) + 1 + for secret in secret_re.finditer(expression_text): + property_name = secret.group(1) + literal_name = secret.group(3) + computed_key = secret.group(4) + if property_name is not None: + name = property_name + display = f"secrets.{name}" + elif literal_name is not None: + name = literal_name + display = f"secrets[{secret.group(2)}{name}{secret.group(2)}]" + else: + key = (computed_key or "").strip() + print(f"{workflow}:{expression_line}: computed secrets key secrets[{key}]") + continue + + if name not in allowed: + print(f"{workflow}:{expression_line}: {display}") +PY + echo "Failed to scan secrets expressions in $workflow" + failed=1 + continue + fi + if [ -s "$disallowed_secrets_file" ]; then + echo "Disallowed secrets expressions found in $workflow:" + cat "$disallowed_secrets_file" + failed=1 + fi + + write_permissions_file="$tmp_dir/write-permissions.txt" + if ! python3 - "$workflow" >"$write_permissions_file" <<'PY'; then +import re +import sys + +workflow = sys.argv[1] + + +def strip_comment(line): + quote = None + escaped = False + for index, char in enumerate(line): + if escaped: + escaped = False + continue + if char == "\\" and quote == '"': + escaped = True + continue + if quote: + if char == quote: + quote = None + continue + if char in {"'", '"'}: + quote = char + continue + if char == "#": + return line[:index] + return line + + +def normalize(value): + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + return value.strip() + + +def flow_pairs(value): + value = value.strip() + if not (value.startswith("{") and value.endswith("}")): + return [] + body = value[1:-1] + parts = [] + start = 0 + quote = None + escaped = False + for index, char in enumerate(body): + if escaped: + escaped = False + continue + if char == "\\" and quote == '"': + escaped = True + continue + if quote: + if char == quote: + quote = None + continue + if char in {"'", '"'}: + quote = char + continue + if char == ",": + parts.append(body[start:index]) + start = index + 1 + parts.append(body[start:]) + pairs = [] + for part in parts: + if ":" not in part: + continue + key, pair_value = part.split(":", 1) + pairs.append((key.strip(), normalize(pair_value))) + return pairs + + +with open(workflow, encoding="utf-8") as f: + lines = f.readlines() + +in_permissions = False +permissions_indent = -1 + +for line_number, line in enumerate(lines, start=1): + without_comment = strip_comment(line).rstrip() + stripped = without_comment.strip() + if in_permissions: + if not stripped: + continue + indent = len(without_comment) - len(without_comment.lstrip(" ")) + if indent <= permissions_indent: + in_permissions = False + else: + match = re.match(r"[A-Za-z0-9_-]+\s*:\s*(.+)$", stripped) + if match and normalize(match.group(1)) == "write": + print(f"{workflow}:{line_number}: {line.rstrip()}") + continue + + match = re.match(r"^(\s*)permissions\s*:\s*(.*)$", without_comment) + if not match: + continue + + permissions_indent = len(match.group(1)) + value = match.group(2).strip() + if not value: + in_permissions = True + continue + + normalized = normalize(value) + if normalized == "write-all": + print(f"{workflow}:{line_number}: {line.rstrip()}") + continue + for _, pair_value in flow_pairs(value): + if pair_value == "write": + print(f"{workflow}:{line_number}: {line.rstrip()}") + break +PY + echo "Failed to scan permissions in $workflow" + failed=1 + continue + fi + if [ -s "$write_permissions_file" ]; then + echo "Write permissions found in $workflow:" + cat "$write_permissions_file" + failed=1 + fi +done + +exit "$failed" diff --git a/scripts/check-cgo-cjs-workflow-purity_test.sh b/scripts/check-cgo-cjs-workflow-purity_test.sh new file mode 100644 index 00000000000..0fedaefe605 --- /dev/null +++ b/scripts/check-cgo-cjs-workflow-purity_test.sh @@ -0,0 +1,133 @@ +#!/bin/bash +set +o histexpand + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PURITY_SCRIPT="$SCRIPT_DIR/check-cgo-cjs-workflow-purity.sh" + +TESTS_PASSED=0 +TESTS_FAILED=0 + +pass() { echo "PASS: $1"; TESTS_PASSED=$((TESTS_PASSED + 1)); } +fail() { echo "FAIL: $1"; echo " $2"; TESTS_FAILED=$((TESTS_FAILED + 1)); } + +write_workflow() { + local path="$1" + local body="$2" + mkdir -p "$(dirname "$path")" + cat > "$path" <"$T1_OUT" 2>&1); then + pass "allowed GITHUB_TOKEN passes in property and bracket syntax" +else + fail "allowed secrets should pass" "$(cat "$T1_OUT")" +fi + +echo "Test 2: forbidden nested and bracket secrets fail..." +T2="$TMP_ROOT/t2" +T2_CGO="$T2/cgo.yml" +T2_CJS="$T2/cjs.yml" +write_workflow "$T2_CGO" " permissions: + contents: read + steps: + - run: echo \"\${{ condition && secrets.DEPLOY_KEY }} \${{ condition && secrets.SCIENCE }}\"" +write_workflow "$T2_CJS" $' permissions:\n contents: read\n steps:\n - run: echo "${{ secrets[\'PROD_TOKEN\'] }} ${{ secrets["PROD_TOKEN_2"] }}"' +T2_OUT="$TMP_ROOT/t2-output.txt" +if (cd "$T2" && bash "$PURITY_SCRIPT" cgo.yml cjs.yml >"$T2_OUT" 2>&1); then + fail "forbidden secrets should exit 1" "$(cat "$T2_OUT")" +elif grep -q "secrets.DEPLOY_KEY" "$T2_OUT" && grep -q "secrets.SCIENCE" "$T2_OUT" && grep -q "PROD_TOKEN" "$T2_OUT" && grep -q "PROD_TOKEN_2" "$T2_OUT"; then + pass "forbidden nested and bracket secrets fail" +else + fail "forbidden secret output was incorrect" "$(cat "$T2_OUT")" +fi + +echo "Test 3: computed secret keys fail..." +T3="$TMP_ROOT/t3" +T3_CGO="$T3/cgo.yml" +write_workflow "$T3_CGO" " permissions: + contents: read + steps: + - run: echo \"\${{ secrets[matrix.secret_name] }}\"" +T3_OUT="$TMP_ROOT/t3-output.txt" +if (cd "$T3" && bash "$PURITY_SCRIPT" cgo.yml >"$T3_OUT" 2>&1); then + fail "computed secret key should exit 1" "$(cat "$T3_OUT")" +elif grep -q "computed secrets key" "$T3_OUT"; then + pass "computed secret keys fail" +else + fail "computed secret output was incorrect" "$(cat "$T3_OUT")" +fi + +echo "Test 4: block, flow, and quoted write permissions fail..." +T4="$TMP_ROOT/t4" +T4_CGO="$T4/block.yml" +T4_CJS="$T4/flow.yml" +T4_SCALAR="$T4/scalar.yml" +write_workflow "$T4_CGO" " permissions: + contents: \"write\" + steps: + - run: echo ok" +write_workflow "$T4_CJS" " permissions: { contents: read, actions: 'write' } + steps: + - run: echo ok" +write_workflow "$T4_SCALAR" " permissions: 'write-all' + steps: + - run: echo ok" +T4_OUT="$TMP_ROOT/t4-output.txt" +if (cd "$T4" && bash "$PURITY_SCRIPT" block.yml flow.yml scalar.yml >"$T4_OUT" 2>&1); then + fail "write permissions should exit 1" "$(cat "$T4_OUT")" +elif grep -q "contents: \"write\"" "$T4_OUT" && grep -q "actions: 'write'" "$T4_OUT" && grep -q "write-all" "$T4_OUT"; then + pass "block, flow, and quoted write permissions fail" +else + fail "write permission output was incorrect" "$(cat "$T4_OUT")" +fi + +echo "Test 5: multiple inputs report missing files while scanning existing files..." +T5="$TMP_ROOT/t5" +T5_CGO="$T5/cgo.yml" +write_workflow "$T5_CGO" " permissions: + contents: read + steps: + - run: echo \"\${{ secrets.DEPLOY_KEY }}\"" +T5_OUT="$TMP_ROOT/t5-output.txt" +if (cd "$T5" && bash "$PURITY_SCRIPT" cgo.yml missing.yml >"$T5_OUT" 2>&1); then + fail "missing file and forbidden secret should exit 1" "$(cat "$T5_OUT")" +elif grep -q "Missing workflow file: missing.yml" "$T5_OUT" && grep -q "secrets.DEPLOY_KEY" "$T5_OUT"; then + pass "multiple inputs report missing files while scanning existing files" +else + fail "multiple input output was incorrect" "$(cat "$T5_OUT")" +fi + +echo +echo "Tests passed: $TESTS_PASSED" +echo "Tests failed: $TESTS_FAILED" + +if [ "$TESTS_FAILED" -gt 0 ]; then + exit 1 +fi + +echo "✓ All tests passed!"