diff --git a/.github/workflows/sync-skills.yml b/.github/workflows/sync-skills.yml index 8e3ccbd6..dd93885f 100644 --- a/.github/workflows/sync-skills.yml +++ b/.github/workflows/sync-skills.yml @@ -254,6 +254,58 @@ jobs: rm -rf "$dir" fi fi + + # Defense #3: signed-but-inconsistent — the rsync delivered a + # sig refresh (with or without content edits), but the incoming + # content does not hash-match the incoming signature. The source + # repo itself is internally inconsistent (skill files edited + # after the signing run, or a pipeline ordering fault). Landing + # it would publish content that fails public signature + # verification — and bot sync PRs run no CI, so this loop is + # the only gate on the ingestion path. Verification mirrors + # .github/scripts/verify_content_integrity.py (digest compare + # of every signed resource; missing files count as mismatches). + if [ -n "$sig_changed" ]; then + mismatch=$(python3 - "$dir" <<'PYEOF' + import base64, hashlib, json, sys + from pathlib import Path + d = Path(sys.argv[1]) + try: + bundle = json.loads((d / "skill.oms.sig").read_text()) + stmt = json.loads(base64.b64decode(bundle["dsseEnvelope"]["payload"])) + resources = stmt["predicate"]["resources"] + except Exception: + # Malformed bundle: treat as one mismatch so the skill is held + # rather than published unverifiable. + print(1) + sys.exit(0) + bad = 0 + for res in resources: + target = d / res["name"] + if not target.is_file(): + bad += 1 + continue + if hashlib.sha256(target.read_bytes()).hexdigest() != res["digest"]: + bad += 1 + print(bad if bad else "") + PYEOF + ) + if [ -n "$mismatch" ]; then + product=$(echo "$dir" | awk -F/ '{print $2}') + if git ls-tree HEAD -- "$dir" | grep -q .; then + reason="sig mismatch: reverted ($mismatch signed file(s) do not match the incoming skill.oms.sig; source repo is internally inconsistent — re-run signing at source)" + echo "$product|$dir|$reason" >> "$dropped" + echo " ↩ $dir — $reason — reverting to HEAD" + git checkout HEAD -- "$dir/" + git clean -fdx -- "$dir/" >/dev/null + else + reason="sig mismatch: dropped ($mismatch signed file(s) do not match skill.oms.sig — new skill, no prior version)" + echo "$product|$dir|$reason" >> "$dropped" + echo " ✗ $dir — $reason" + rm -rf "$dir" + fi + fi + fi done < /tmp/rsynced-skill-dirs.txt # Mark any product with a dropped skill as "changed" so the PR @@ -526,8 +578,9 @@ jobs: missing_count=$(awk -F'|' '$3 ~ /^missing artifacts:/' "$dropped" | wc -l | tr -d ' ') orphan_count=$(awk -F'|' '$3 ~ /^orphan:/' "$dropped" | wc -l | tr -d ' ') drift_count=$(awk -F'|' '$3 ~ /^sig drift:/' "$dropped" | wc -l | tr -d ' ') + mismatch_count=$(awk -F'|' '$3 ~ /^sig mismatch:/' "$dropped" | wc -l | tr -d ' ') nosig_count=$(awk -F'|' '$3 ~ /^sig missing:/' "$dropped" | wc -l | tr -d ' ') - other_count=$(awk -F'|' '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig missing:)/' "$dropped" | wc -l | tr -d ' ') + other_count=$(awk -F'|' '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig mismatch:|sig missing:)/' "$dropped" | wc -l | tr -d ' ') body=/tmp/missing-compliance-body.md { @@ -541,11 +594,13 @@ jobs: echo " - Skills that previously existed on \`main\` are **reverted** to their last-signed version (the signed copy stays live)." echo " - Brand-new skills with drift are **dropped** outright." echo " Recovery: comment \`/nvskills-ci\` on a follow-up PR on the source repo so the next sync picks up a fresh, content-aligned signature." - echo "3. **Signature missing** — source skill dir has no \`skill.oms.sig\` at all (never signed, or sig deleted upstream):" + echo "3. **Signature mismatch** — the source repo delivered a refreshed \`skill.oms.sig\`, but the skill's content does not hash-match it. The source repo is internally inconsistent (skill files edited after the signing run, or a pipeline ordering fault). Same revert/drop handling as drift." + echo " Recovery: re-run \`/nvskills-ci\` at source and do not modify skill files after the generated signature commit lands." + echo "4. **Signature missing** — source skill dir has no \`skill.oms.sig\` at all (never signed, or sig deleted upstream):" echo " - Previously-signed catalog skills are **reverted** to their last-signed version." echo " - New unsigned skills are **dropped** outright." echo " Recovery: run the signing flow on the source repo so \`skill.oms.sig\` lands alongside \`SKILL.md\`." - echo "4. **Orphan product dirs** — product folder under \`skills/\` with no \`SKILL.md\` inside (e.g. every skill was dropped, leaving only helper files)." + echo "5. **Orphan product dirs** — product folder under \`skills/\` with no \`SKILL.md\` inside (e.g. every skill was dropped, leaving only helper files)." echo "" echo "**Last updated:** $(date -u +%Y-%m-%d) by [sync run #${{ github.run_id }}]($RUN_URL)" echo "" @@ -571,8 +626,9 @@ jobs: case "$reason" in "missing artifacts: "*) kind="missing"; display="${reason#missing artifacts: }" ;; "orphan: "*) kind="orphan"; display="${reason#orphan: }" ;; - "sig drift: "*) kind="drift"; display="${reason#sig drift: }" ;; - "sig missing: "*) kind="nosig"; display="${reason#sig missing: }" ;; + "sig drift: "*) kind="drift"; display="${reason#sig drift: }" ;; + "sig mismatch: "*) kind="mismatch"; display="${reason#sig mismatch: }" ;; + "sig missing: "*) kind="nosig"; display="${reason#sig missing: }" ;; *) kind="other"; display="$reason" ;; esac [ "$kind" = "$filter_kind" ] || continue @@ -592,11 +648,12 @@ jobs: echo "" } - emit_section "Missing artifacts" "missing" "$missing_count" - emit_section "Signature drift" "drift" "$drift_count" - emit_section "Signature missing" "nosig" "$nosig_count" - emit_section "Orphan product dirs" "orphan" "$orphan_count" - emit_section "Other" "other" "$other_count" + emit_section "Missing artifacts" "missing" "$missing_count" + emit_section "Signature drift" "drift" "$drift_count" + emit_section "Signature mismatch" "mismatch" "$mismatch_count" + emit_section "Signature missing" "nosig" "$nosig_count" + emit_section "Orphan product dirs" "orphan" "$orphan_count" + emit_section "Other" "other" "$other_count" echo "---" echo "Owners: re-run \`/nvskills-ci\` on the source repo after editing skill content so the signature stays in sync. This issue auto-updates each sync and closes when all skills are compliant." @@ -606,11 +663,12 @@ jobs: # whichever buckets are non-empty so it stays accurate as # categories come and go. parts=() - [ "$missing_count" -gt 0 ] && parts+=("$missing_count missing artifacts") - [ "$drift_count" -gt 0 ] && parts+=("$drift_count sig drift") - [ "$nosig_count" -gt 0 ] && parts+=("$nosig_count sig missing") - [ "$orphan_count" -gt 0 ] && parts+=("$orphan_count orphan") - [ "$other_count" -gt 0 ] && parts+=("$other_count other") + [ "$missing_count" -gt 0 ] && parts+=("$missing_count missing artifacts") + [ "$drift_count" -gt 0 ] && parts+=("$drift_count sig drift") + [ "$mismatch_count" -gt 0 ] && parts+=("$mismatch_count sig mismatch") + [ "$nosig_count" -gt 0 ] && parts+=("$nosig_count sig missing") + [ "$orphan_count" -gt 0 ] && parts+=("$orphan_count orphan") + [ "$other_count" -gt 0 ] && parts+=("$other_count other") if [ ${#parts[@]} -eq 0 ]; then title="Skills dropped from sync" # shouldn't happen (we exit 0 above) else @@ -798,9 +856,10 @@ jobs: # step so the PR body and the rolling issue stay consistent. missing_lines=$(awk -F'|' '$3 ~ /^missing artifacts:/' /tmp/dropped-skills.txt || true) drift_lines=$(awk -F'|' '$3 ~ /^sig drift:/' /tmp/dropped-skills.txt || true) + mismatch_lines=$(awk -F'|' '$3 ~ /^sig mismatch:/' /tmp/dropped-skills.txt || true) nosig_lines=$(awk -F'|' '$3 ~ /^sig missing:/' /tmp/dropped-skills.txt || true) orphan_lines=$(awk -F'|' '$3 ~ /^orphan:/' /tmp/dropped-skills.txt || true) - other_lines=$(awk -F'|' '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig missing:)/' /tmp/dropped-skills.txt || true) + other_lines=$(awk -F'|' '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig mismatch:|sig missing:)/' /tmp/dropped-skills.txt || true) # Lookup helper mirrors the tracker-issue step: append a # `cc: @` trailer per skill when /tmp/skill-contacts.txt @@ -836,6 +895,13 @@ jobs: emit_with_contact "$dir" "${reason#sig drift: }" done fi + if [ -n "$mismatch_lines" ]; then + echo "" + echo "**Skills held — signature mismatch (incoming content does not match its own \`skill.oms.sig\`):**" + echo "$mismatch_lines" | while IFS='|' read -r _ dir reason; do + emit_with_contact "$dir" "${reason#sig mismatch: }" + done + fi if [ -n "$nosig_lines" ]; then echo "" echo "**Skills held — no signature in source (\`skill.oms.sig\` absent upstream):**" diff --git a/.github/workflows/verify-content-integrity.yml b/.github/workflows/verify-content-integrity.yml index 1c50eab9..45b314f6 100644 --- a/.github/workflows/verify-content-integrity.yml +++ b/.github/workflows/verify-content-integrity.yml @@ -15,6 +15,11 @@ # drift being introduced without blocking on unrelated pre-existing state). On # a schedule / manual run it checks the whole catalog as a safety net for # drift already on main. +# +# Scheduled/manual failures are surfaced in a rolling tracker issue +# (label: integrity-failure) — a silently red scheduled run alerts no one, +# which is how published mismatches sat unnoticed for a week in July 2026. +# The issue auto-closes when a sweep comes back clean. name: Verify Content Integrity @@ -28,6 +33,7 @@ on: permissions: contents: read + issues: write jobs: content-integrity: @@ -41,8 +47,80 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Verify content matches signatures + id: verify env: GITHUB_EVENT_NAME: ${{ github.event_name }} BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: python3 .github/scripts/verify_content_integrity.py + run: | + set -o pipefail + python3 .github/scripts/verify_content_integrity.py 2>&1 \ + | tee /tmp/integrity-report.txt + + - name: Update integrity tracker issue + # Full-catalog runs only (schedule / manual). PR failures are + # visible on the PR itself; sweep failures otherwise alert no one. + # always() so this runs whether the sweep passed (close tracker) + # or failed (open/update tracker). + if: ${{ always() && github.event_name != 'pull_request' }} + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SWEEP_OUTCOME: ${{ steps.verify.outcome }} + run: | + set -euo pipefail + + gh label create integrity-failure \ + --color D93F0B \ + --description "Published catalog content fails signature verification (daily sweep)" \ + --force >/dev/null + + existing=$(gh issue list --label integrity-failure --state open \ + --json number --jq '.[0].number // empty' --limit 1) + + if [ "$SWEEP_OUTCOME" = "success" ]; then + if [ -n "$existing" ]; then + gh issue close "$existing" \ + --comment "Daily integrity sweep is clean — all published skill content matches its signatures again. Closing automatically. ([run]($RUN_URL))" + echo "Sweep clean — closed tracking issue #$existing" + else + echo "Sweep clean — nothing to track" + fi + exit 0 + fi + + # Failed sweep: build the tracker body from the report's + # finding lines. If the report file is missing (step crashed + # before producing output), say so rather than posting empty. + findings=$(grep -E "MISMATCH|MISSING|EXTRA|INVALID|could not parse" /tmp/integrity-report.txt || true) + finding_count=$(printf '%s' "$findings" | grep -c . || true) + + body=/tmp/integrity-issue-body.md + { + echo "The daily full-catalog integrity sweep is failing: published skill content does not match its \`skill.oms.sig\`. Anyone verifying our signatures with public tooling gets failures on these files today." + echo "" + echo "**Last updated:** $(date -u +%Y-%m-%d) by [sweep run]($RUN_URL) — $finding_count finding(s)" + echo "" + echo '```' + if [ -n "$findings" ]; then + echo "$findings" + else + echo "(report unavailable — see the run log)" + fi + echo '```' + echo "" + echo "**Recovery (skill owners):** the mismatch exists at your source repo — content was modified after the signing run. Open a PR that touches the skill directory (a trivial SKILL.md version bump works if your changes are already merged), comment \`/nvskills-ci\`, and merge with the generated signature commit. Do not modify skill files afterward. Steps: https://nvidia.atlassian.net/wiki/spaces/GAIT/pages/3483240468" + echo "" + echo "This issue auto-updates on every failing sweep and closes automatically when a sweep passes." + } > "$body" + + title="Integrity sweep failing — $finding_count file(s) do not match their signatures" + + if [ -n "$existing" ]; then + gh issue edit "$existing" --title "$title" --body-file "$body" + echo "Updated tracking issue #$existing" + else + gh issue create --title "$title" --body-file "$body" --label integrity-failure + echo "Opened new tracking issue" + fi