Hold sig-mismatched skills at sync; alert on failing integrity sweeps

.github/workflows/sync-skills.yml

@@ -254,6 +254,58 @@ jobs:
                 rm -rf "$dir"
               fi
             fi
+
+            # Defense #3: signed-but-inconsistent — the rsync delivered a
+            # sig refresh (with or without content edits), but the incoming
+            # content does not hash-match the incoming signature. The source
+            # repo itself is internally inconsistent (skill files edited
+            # after the signing run, or a pipeline ordering fault). Landing
+            # it would publish content that fails public signature
+            # verification — and bot sync PRs run no CI, so this loop is
+            # the only gate on the ingestion path. Verification mirrors
+            # .github/scripts/verify_content_integrity.py (digest compare
+            # of every signed resource; missing files count as mismatches).
+            if [ -n "$sig_changed" ]; then
+              mismatch=$(python3 - "$dir" <<'PYEOF'
+          import base64, hashlib, json, sys
+          from pathlib import Path
+          d = Path(sys.argv[1])
+          try:
+              bundle = json.loads((d / "skill.oms.sig").read_text())
+              stmt = json.loads(base64.b64decode(bundle["dsseEnvelope"]["payload"]))
+              resources = stmt["predicate"]["resources"]
+          except Exception:
+              # Malformed bundle: treat as one mismatch so the skill is held
+              # rather than published unverifiable.
+              print(1)
+              sys.exit(0)
+          bad = 0
+          for res in resources:
+              target = d / res["name"]
+              if not target.is_file():
+                  bad += 1
+                  continue
+              if hashlib.sha256(target.read_bytes()).hexdigest() != res["digest"]:
+                  bad += 1
+          print(bad if bad else "")
+          PYEOF
+              )
+              if [ -n "$mismatch" ]; then
+                product=$(echo "$dir" | awk -F/ '{print $2}')
+                if git ls-tree HEAD -- "$dir" | grep -q .; then
+                  reason="sig mismatch: reverted ($mismatch signed file(s) do not match the incoming skill.oms.sig; source repo is internally inconsistent — re-run signing at source)"
+                  echo "$product|$dir|$reason" >> "$dropped"
+                  echo "  ↩ $dir — $reason — reverting to HEAD"
+                  git checkout HEAD -- "$dir/"
+                  git clean -fdx -- "$dir/" >/dev/null
+                else
+                  reason="sig mismatch: dropped ($mismatch signed file(s) do not match skill.oms.sig — new skill, no prior version)"
+                  echo "$product|$dir|$reason" >> "$dropped"
+                  echo "  ✗ $dir — $reason"
+                  rm -rf "$dir"
+                fi
+              fi
+            fi
           done < /tmp/rsynced-skill-dirs.txt
 
           # Mark any product with a dropped skill as "changed" so the PR
@@ -526,8 +578,9 @@ jobs:
           missing_count=$(awk -F'|' '$3 ~ /^missing artifacts:/' "$dropped" | wc -l | tr -d ' ')
           orphan_count=$(awk -F'|' '$3 ~ /^orphan:/' "$dropped" | wc -l | tr -d ' ')
           drift_count=$(awk -F'|' '$3 ~ /^sig drift:/' "$dropped" | wc -l | tr -d ' ')
+          mismatch_count=$(awk -F'|' '$3 ~ /^sig mismatch:/' "$dropped" | wc -l | tr -d ' ')
           nosig_count=$(awk -F'|' '$3 ~ /^sig missing:/' "$dropped" | wc -l | tr -d ' ')
-          other_count=$(awk -F'|' '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig missing:)/' "$dropped" | wc -l | tr -d ' ')
+          other_count=$(awk -F'|' '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig mismatch:|sig missing:)/' "$dropped" | wc -l | tr -d ' ')
 
           body=/tmp/missing-compliance-body.md
           {
@@ -541,11 +594,13 @@ jobs:
             echo "   - Skills that previously existed on \`main\` are **reverted** to their last-signed version (the signed copy stays live)."
             echo "   - Brand-new skills with drift are **dropped** outright."
             echo "   Recovery: comment \`/nvskills-ci\` on a follow-up PR on the source repo so the next sync picks up a fresh, content-aligned signature."
-            echo "3. **Signature missing** — source skill dir has no \`skill.oms.sig\` at all (never signed, or sig deleted upstream):"
+            echo "3. **Signature mismatch** — the source repo delivered a refreshed \`skill.oms.sig\`, but the skill's content does not hash-match it. The source repo is internally inconsistent (skill files edited after the signing run, or a pipeline ordering fault). Same revert/drop handling as drift."
+            echo "   Recovery: re-run \`/nvskills-ci\` at source and do not modify skill files after the generated signature commit lands."
+            echo "4. **Signature missing** — source skill dir has no \`skill.oms.sig\` at all (never signed, or sig deleted upstream):"
             echo "   - Previously-signed catalog skills are **reverted** to their last-signed version."
             echo "   - New unsigned skills are **dropped** outright."
             echo "   Recovery: run the signing flow on the source repo so \`skill.oms.sig\` lands alongside \`SKILL.md\`."
-            echo "4. **Orphan product dirs** — product folder under \`skills/\` with no \`SKILL.md\` inside (e.g. every skill was dropped, leaving only helper files)."
+            echo "5. **Orphan product dirs** — product folder under \`skills/\` with no \`SKILL.md\` inside (e.g. every skill was dropped, leaving only helper files)."
             echo ""
             echo "**Last updated:** $(date -u +%Y-%m-%d) by [sync run #${{ github.run_id }}]($RUN_URL)"
             echo ""
@@ -571,8 +626,9 @@ jobs:
                 case "$reason" in
                   "missing artifacts: "*) kind="missing"; display="${reason#missing artifacts: }" ;;
                   "orphan: "*)            kind="orphan";  display="${reason#orphan: }" ;;
-                  "sig drift: "*)         kind="drift";   display="${reason#sig drift: }" ;;
-                  "sig missing: "*)       kind="nosig";   display="${reason#sig missing: }" ;;
+                  "sig drift: "*)         kind="drift";    display="${reason#sig drift: }" ;;
+                  "sig mismatch: "*)      kind="mismatch"; display="${reason#sig mismatch: }" ;;
+                  "sig missing: "*)       kind="nosig";    display="${reason#sig missing: }" ;;
                   *)                      kind="other";   display="$reason" ;;
                 esac
                 [ "$kind" = "$filter_kind" ] || continue
@@ -592,11 +648,12 @@ jobs:
               echo ""
             }
 
-            emit_section "Missing artifacts"   "missing" "$missing_count"
-            emit_section "Signature drift"     "drift"   "$drift_count"
-            emit_section "Signature missing"   "nosig"   "$nosig_count"
-            emit_section "Orphan product dirs" "orphan"  "$orphan_count"
-            emit_section "Other"               "other"   "$other_count"
+            emit_section "Missing artifacts"   "missing"  "$missing_count"
+            emit_section "Signature drift"     "drift"    "$drift_count"
+            emit_section "Signature mismatch"  "mismatch" "$mismatch_count"
+            emit_section "Signature missing"   "nosig"    "$nosig_count"
+            emit_section "Orphan product dirs" "orphan"   "$orphan_count"
+            emit_section "Other"               "other"    "$other_count"
 
             echo "---"
             echo "Owners: re-run \`/nvskills-ci\` on the source repo after editing skill content so the signature stays in sync. This issue auto-updates each sync and closes when all skills are compliant."
@@ -606,11 +663,12 @@ jobs:
           # whichever buckets are non-empty so it stays accurate as
           # categories come and go.
           parts=()
-          [ "$missing_count" -gt 0 ] && parts+=("$missing_count missing artifacts")
-          [ "$drift_count"   -gt 0 ] && parts+=("$drift_count sig drift")
-          [ "$nosig_count"   -gt 0 ] && parts+=("$nosig_count sig missing")
-          [ "$orphan_count"  -gt 0 ] && parts+=("$orphan_count orphan")
-          [ "$other_count"   -gt 0 ] && parts+=("$other_count other")
+          [ "$missing_count"  -gt 0 ] && parts+=("$missing_count missing artifacts")
+          [ "$drift_count"    -gt 0 ] && parts+=("$drift_count sig drift")
+          [ "$mismatch_count" -gt 0 ] && parts+=("$mismatch_count sig mismatch")
+          [ "$nosig_count"    -gt 0 ] && parts+=("$nosig_count sig missing")
+          [ "$orphan_count"   -gt 0 ] && parts+=("$orphan_count orphan")
+          [ "$other_count"    -gt 0 ] && parts+=("$other_count other")
           if [ ${#parts[@]} -eq 0 ]; then
             title="Skills dropped from sync"   # shouldn't happen (we exit 0 above)
           else
@@ -798,9 +856,10 @@ jobs:
               # step so the PR body and the rolling issue stay consistent.
               missing_lines=$(awk -F'|' '$3 ~ /^missing artifacts:/' /tmp/dropped-skills.txt || true)
               drift_lines=$(awk -F'|'   '$3 ~ /^sig drift:/'         /tmp/dropped-skills.txt || true)
+              mismatch_lines=$(awk -F'|' '$3 ~ /^sig mismatch:/'     /tmp/dropped-skills.txt || true)
               nosig_lines=$(awk -F'|'   '$3 ~ /^sig missing:/'       /tmp/dropped-skills.txt || true)
               orphan_lines=$(awk -F'|'  '$3 ~ /^orphan:/'            /tmp/dropped-skills.txt || true)
-              other_lines=$(awk -F'|'   '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig missing:)/' /tmp/dropped-skills.txt || true)
+              other_lines=$(awk -F'|'   '$3 !~ /^(missing artifacts:|orphan:|sig drift:|sig mismatch:|sig missing:)/' /tmp/dropped-skills.txt || true)
 
               # Lookup helper mirrors the tracker-issue step: append a
               # `cc: @<login>` trailer per skill when /tmp/skill-contacts.txt
@@ -836,6 +895,13 @@ jobs:
                   emit_with_contact "$dir" "${reason#sig drift: }"
                 done
               fi
+              if [ -n "$mismatch_lines" ]; then
+                echo ""
+                echo "**Skills held — signature mismatch (incoming content does not match its own \`skill.oms.sig\`):**"
+                echo "$mismatch_lines" | while IFS='|' read -r _ dir reason; do
+                  emit_with_contact "$dir" "${reason#sig mismatch: }"
+                done
+              fi
               if [ -n "$nosig_lines" ]; then
                 echo ""
                 echo "**Skills held — no signature in source (\`skill.oms.sig\` absent upstream):**"

.github/workflows/verify-content-integrity.yml

@@ -15,6 +15,11 @@
 # drift being introduced without blocking on unrelated pre-existing state). On
 # a schedule / manual run it checks the whole catalog as a safety net for
 # drift already on main.
+#
+# Scheduled/manual failures are surfaced in a rolling tracker issue
+# (label: integrity-failure) — a silently red scheduled run alerts no one,
+# which is how published mismatches sat unnoticed for a week in July 2026.
+# The issue auto-closes when a sweep comes back clean.
 
 name: Verify Content Integrity
 
@@ -28,6 +33,7 @@ on:
 
 permissions:
   contents: read
+  issues: write
 
 jobs:
   content-integrity:
@@ -41,8 +47,80 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Verify content matches signatures
+        id: verify
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: python3 .github/scripts/verify_content_integrity.py
+        run: |
+          set -o pipefail
+          python3 .github/scripts/verify_content_integrity.py 2>&1 \
+            | tee /tmp/integrity-report.txt
+
+      - name: Update integrity tracker issue
+        # Full-catalog runs only (schedule / manual). PR failures are
+        # visible on the PR itself; sweep failures otherwise alert no one.
+        # always() so this runs whether the sweep passed (close tracker)
+        # or failed (open/update tracker).
+        if: ${{ always() && github.event_name != 'pull_request' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SWEEP_OUTCOME: ${{ steps.verify.outcome }}
+        run: |
+          set -euo pipefail
+
+          gh label create integrity-failure \
+            --color D93F0B \
+            --description "Published catalog content fails signature verification (daily sweep)" \
+            --force >/dev/null
+
+          existing=$(gh issue list --label integrity-failure --state open \
+                       --json number --jq '.[0].number // empty' --limit 1)
+
+          if [ "$SWEEP_OUTCOME" = "success" ]; then
+            if [ -n "$existing" ]; then
+              gh issue close "$existing" \
+                --comment "Daily integrity sweep is clean — all published skill content matches its signatures again. Closing automatically. ([run]($RUN_URL))"
+              echo "Sweep clean — closed tracking issue #$existing"
+            else
+              echo "Sweep clean — nothing to track"
+            fi
+            exit 0
+          fi
+
+          # Failed sweep: build the tracker body from the report's
+          # finding lines. If the report file is missing (step crashed
+          # before producing output), say so rather than posting empty.
+          findings=$(grep -E "MISMATCH|MISSING|EXTRA|INVALID|could not parse" /tmp/integrity-report.txt || true)
+          finding_count=$(printf '%s' "$findings" | grep -c . || true)
+
+          body=/tmp/integrity-issue-body.md
+          {
+            echo "The daily full-catalog integrity sweep is failing: published skill content does not match its \`skill.oms.sig\`. Anyone verifying our signatures with public tooling gets failures on these files today."
+            echo ""
+            echo "**Last updated:** $(date -u +%Y-%m-%d) by [sweep run]($RUN_URL) — $finding_count finding(s)"
+            echo ""
+            echo '```'
+            if [ -n "$findings" ]; then
+              echo "$findings"
+            else
+              echo "(report unavailable — see the run log)"
+            fi
+            echo '```'
+            echo ""
+            echo "**Recovery (skill owners):** the mismatch exists at your source repo — content was modified after the signing run. Open a PR that touches the skill directory (a trivial SKILL.md version bump works if your changes are already merged), comment \`/nvskills-ci\`, and merge with the generated signature commit. Do not modify skill files afterward. Steps: https://nvidia.atlassian.net/wiki/spaces/GAIT/pages/3483240468"
+            echo ""
+            echo "This issue auto-updates on every failing sweep and closes automatically when a sweep passes."
+          } > "$body"
+
+          title="Integrity sweep failing — $finding_count file(s) do not match their signatures"
+
+          if [ -n "$existing" ]; then
+            gh issue edit "$existing" --title "$title" --body-file "$body"
+            echo "Updated tracking issue #$existing"
+          else
+            gh issue create --title "$title" --body-file "$body" --label integrity-failure
+            echo "Opened new tracking issue"
+          fi