Update audit log event data (#61769)

docs-bot · saritai · mchammer01 · web-flow · commit ecd8bed1fe0f · 2026-06-22T15:15:59.000Z
Co-authored-by: Sarita Iyer &lt;66540150+saritai@users.noreply.github.com&gt;
Co-authored-by: mc &lt;42146119+mchammer01@users.noreply.github.com&gt;
Co-authored-by: Kevin Heis &lt;heiskr@users.noreply.github.com&gt;
Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/package.json b/package.json
@@ -97,6 +97,7 @@
     "start-for-playwright": "cross-env ROOT=src/fixtures/fixtures TRANSLATIONS_FIXTURE_ROOT=src/fixtures/fixtures/translations ENABLED_LANGUAGES=en,ja NODE_ENV=test tsx src/frame/server.ts",
     "symlink-from-local-repo": "tsx src/early-access/scripts/symlink-from-local-repo.ts",
     "sync-audit-log": "tsx src/audit-logs/scripts/sync.ts",
+    "rebuild-audit-log-dedup": "tsx src/audit-logs/scripts/rebuild-dedup.ts",
     "sync-codeql-cli": "tsx src/codeql-cli/scripts/sync.ts",
     "sync-graphql": "tsx src/graphql/scripts/sync.ts",
     "sync-rest": "tsx src/rest/scripts/update-files.ts",
diff --git a/src/audit-logs/data/fpt/organization.json b/src/audit-logs/data/fpt/organization.json
@@ -18023,6 +18023,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghec/enterprise.json b/src/audit-logs/data/ghec/enterprise.json
@@ -23441,6 +23441,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghec/organization.json b/src/audit-logs/data/ghec/organization.json
@@ -18023,6 +18023,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.17/enterprise.json b/src/audit-logs/data/ghes-3.17/enterprise.json
@@ -17456,6 +17456,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.17/organization.json b/src/audit-logs/data/ghes-3.17/organization.json
@@ -16781,6 +16781,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.18/enterprise.json b/src/audit-logs/data/ghes-3.18/enterprise.json
@@ -17526,6 +17526,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.18/organization.json b/src/audit-logs/data/ghes-3.18/organization.json
@@ -17061,6 +17061,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.19/enterprise.json b/src/audit-logs/data/ghes-3.19/enterprise.json
@@ -18483,6 +18483,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.19/organization.json b/src/audit-logs/data/ghes-3.19/organization.json
@@ -17774,6 +17774,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.20/enterprise.json b/src/audit-logs/data/ghes-3.20/enterprise.json
@@ -18516,6 +18516,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/ghes-3.20/organization.json b/src/audit-logs/data/ghes-3.20/organization.json
@@ -17711,6 +17711,7 @@
       "query_suite",
       "threat_model",
       "languages",
+      "languages_removed",
       "public_repo",
       "created_at",
       "operation_type",
diff --git a/src/audit-logs/data/shared/entries.json b/src/audit-logs/data/shared/entries.json
diff --git a/src/audit-logs/data/shared/fields-pool.json b/src/audit-logs/data/shared/fields-pool.json
diff --git a/src/audit-logs/data/version-index.json b/src/audit-logs/data/version-index.json
diff --git a/src/audit-logs/lib/deduplicate.ts b/src/audit-logs/lib/deduplicate.ts
@@ -0,0 +1,96 @@
+import { existsSync } from 'fs'
+import { writeFile } from 'fs/promises'
+import { mkdirp } from 'mkdirp'
+import path from 'path'
+
+import type {
+  AuditLogEventT,
+  VersionedAuditLogData,
+  DeduplicatedAuditLogEntry,
+  AuditLogVersionIndex,
+} from '../types'
+
+// Kept in sync with the same constant in lib/index.ts. Defined locally here so
+// this write-path module doesn't pull in the request-path lib/index.ts.
+const AUDIT_LOG_DATA_DIR = 'src/audit-logs/data'
+
+// Builds the deduplicated "shared" format (entries pool + fields pool +
+// version index) from in-memory audit log data and writes it to disk.
+//
+// Output is deterministic: versions and pages are iterated in sorted order so
+// that the same input always produces byte-identical files. This lets the sync
+// pipeline (sync.ts) and the standalone rebuild script (rebuild-dedup.ts)
+// produce the exact same files, so the two can never drift apart.
+export async function writeDeduplicatedAuditLogData(
+  auditLogData: VersionedAuditLogData,
+): Promise<void> {
+  console.log(`\n▶️  Writing deduplicated audit log data...\n`)
+
+  // Build fields pool: unique fields arrays
+  const fieldsPool: string[][] = []
+  const fieldsMap = new Map<string, number>() // JSON key → index
+
+  function getFieldsIndex(fields: string[] | undefined): number | undefined {
+    if (!fields || fields.length === 0) return undefined
+    const key = JSON.stringify(fields)
+    if (fieldsMap.has(key)) return fieldsMap.get(key)!
+    const index = fieldsPool.length
+    fieldsPool.push(fields)
+    fieldsMap.set(key, index)
+    return index
+  }
+
+  // Build entries pool: unique events (with fields replaced by index)
+  const entriesPool: DeduplicatedAuditLogEntry[] = []
+  const entriesMap = new Map<string, number>() // JSON key → index
+
+  function getEntryIndex(event: AuditLogEventT): number {
+    const fieldsIndex = getFieldsIndex(event.fields)
+    const entry: DeduplicatedAuditLogEntry = {
+      action: event.action,
+      description: event.description,
+    }
+    if (event.docs_reference_links) entry.docs_reference_links = event.docs_reference_links
+    if (event.docs_reference_titles) entry.docs_reference_titles = event.docs_reference_titles
+    if (fieldsIndex !== undefined) entry.fieldsIndex = fieldsIndex
+
+    const key = JSON.stringify(entry)
+    if (entriesMap.has(key)) return entriesMap.get(key)!
+    const index = entriesPool.length
+    entriesPool.push(entry)
+    entriesMap.set(key, index)
+    return index
+  }
+
+  // Build version index. Iterate versions and pages in sorted order so the
+  // output is deterministic regardless of the input's insertion order.
+  const versionIndex: AuditLogVersionIndex = {}
+  let totalEntries = 0
+
+  for (const version of Object.keys(auditLogData).sort()) {
+    versionIndex[version] = {}
+    const pages = auditLogData[version]
+    for (const page of Object.keys(pages).sort()) {
+      const events = pages[page]
+      versionIndex[version][page] = events.map((event) => getEntryIndex(event))
+      totalEntries += events.length
+    }
+  }
+
+  // Write shared files
+  const sharedDir = path.join(AUDIT_LOG_DATA_DIR, 'shared')
+  if (!existsSync(sharedDir)) {
+    await mkdirp(sharedDir)
+  }
+
+  await writeFile(path.join(sharedDir, 'entries.json'), JSON.stringify(entriesPool))
+  await writeFile(path.join(sharedDir, 'fields-pool.json'), JSON.stringify(fieldsPool))
+  await writeFile(path.join(AUDIT_LOG_DATA_DIR, 'version-index.json'), JSON.stringify(versionIndex))
+
+  const uniqueEntries = entriesPool.length
+  const uniqueFields = fieldsPool.length
+  const dedupRate = totalEntries > 0 ? ((1 - uniqueEntries / totalEntries) * 100).toFixed(1) : '0'
+  console.log(
+    `✅ Deduplicated audit log data: ${totalEntries} total → ${uniqueEntries} unique entries (${dedupRate}% dedup), ${uniqueFields} unique field lists`,
+  )
+}
diff --git a/src/audit-logs/scripts/rebuild-dedup.ts b/src/audit-logs/scripts/rebuild-dedup.ts
@@ -0,0 +1,63 @@
+/**
+ * Rebuilds the deduplicated "shared" audit log format
+ * (src/audit-logs/data/shared/entries.json, fields-pool.json, and
+ * version-index.json) from the per-version page files already on disk.
+ *
+ * Unlike `sync-audit-log`, this does NOT fetch from github/audit-log-allowlists
+ * and needs no GITHUB_TOKEN. Use it to regenerate the shared files when the
+ * per-version JSON files have changed but the shared files are stale (for
+ * example, an older audit-log-pipeline PR that predates the dedup format), which
+ * makes the `deduplication` test fail.
+ *
+ *   npm run rebuild-audit-log-dedup
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from 'fs'
+import path from 'path'
+
+import { writeDeduplicatedAuditLogData } from '../lib/deduplicate'
+import type { AuditLogEventT, VersionedAuditLogData } from '../types'
+
+const AUDIT_LOG_DATA_DIR = 'src/audit-logs/data'
+
+// Directories under the data dir that are not per-version event data.
+const NON_VERSION_DIRS = new Set(['shared'])
+
+function loadAuditLogDataFromDisk(): VersionedAuditLogData {
+  const auditLogData: VersionedAuditLogData = {}
+
+  for (const version of readdirSync(AUDIT_LOG_DATA_DIR)) {
+    if (NON_VERSION_DIRS.has(version)) continue
+    const versionDir = path.join(AUDIT_LOG_DATA_DIR, version)
+    if (!statSync(versionDir).isDirectory()) continue
+
+    const pages: Record<string, AuditLogEventT[]> = {}
+    for (const file of readdirSync(versionDir)) {
+      if (!file.endsWith('.json')) continue
+      const page = path.basename(file, '.json')
+      pages[page] = JSON.parse(readFileSync(path.join(versionDir, file), 'utf8'))
+    }
+
+    if (Object.keys(pages).length > 0) {
+      auditLogData[version] = pages
+    }
+  }
+
+  return auditLogData
+}
+
+async function main() {
+  if (!existsSync(AUDIT_LOG_DATA_DIR)) {
+    throw new Error(`Audit log data directory not found: ${AUDIT_LOG_DATA_DIR}`)
+  }
+
+  const auditLogData = loadAuditLogDataFromDisk()
+  const versionCount = Object.keys(auditLogData).length
+  if (versionCount === 0) {
+    throw new Error(`No per-version audit log data found in ${AUDIT_LOG_DATA_DIR}`)
+  }
+
+  console.log(`\n▶️  Rebuilding deduplicated format from ${versionCount} versions on disk...`)
+  await writeDeduplicatedAuditLogData(auditLogData)
+}
+
+main()
diff --git a/src/audit-logs/scripts/sync.ts b/src/audit-logs/scripts/sync.ts
@@ -13,16 +13,12 @@ import { mkdirp } from 'mkdirp'
 import path from 'path'
 
 import { filterByAllowlistValues, filterAndUpdateGhesDataByAllowlistValues } from '../lib/index'
+import { writeDeduplicatedAuditLogData } from '../lib/deduplicate'
 import { getContents, getCommitSha } from '@/workflows/git-utils'
 import { latest, latestStable, releaseCandidate } from '@/versions/lib/enterprise-server-releases'
 import { loadPages, loadPageMap } from '@/frame/lib/page-data'
 import loadRedirects from '@/redirects/lib/precompile'
-import type {
-  AuditLogEventT,
-  VersionedAuditLogData,
-  DeduplicatedAuditLogEntry,
-  AuditLogVersionIndex,
-} from '../types'
+import type { AuditLogEventT, VersionedAuditLogData } from '../types'
 
 if (!process.env.GITHUB_TOKEN) {
   throw new Error('GITHUB_TOKEN environment variable must be set to run this script')
@@ -216,76 +212,7 @@ async function main() {
   }
 
   // Write deduplicated shared format
-  await writeDeduplicatedFormat(auditLogData)
-}
-
-async function writeDeduplicatedFormat(auditLogData: VersionedAuditLogData) {
-  console.log(`\n▶️  Writing deduplicated audit log data...\n`)
-
-  // Build fields pool: unique fields arrays
-  const fieldsPool: string[][] = []
-  const fieldsMap = new Map<string, number>() // JSON key → index
-
-  function getFieldsIndex(fields: string[] | undefined): number | undefined {
-    if (!fields || fields.length === 0) return undefined
-    const key = JSON.stringify(fields)
-    if (fieldsMap.has(key)) return fieldsMap.get(key)!
-    const index = fieldsPool.length
-    fieldsPool.push(fields)
-    fieldsMap.set(key, index)
-    return index
-  }
-
-  // Build entries pool: unique events (with fields replaced by index)
-  const entriesPool: DeduplicatedAuditLogEntry[] = []
-  const entriesMap = new Map<string, number>() // JSON key → index
-
-  function getEntryIndex(event: AuditLogEventT): number {
-    const fieldsIndex = getFieldsIndex(event.fields)
-    const entry: DeduplicatedAuditLogEntry = {
-      action: event.action,
-      description: event.description,
-    }
-    if (event.docs_reference_links) entry.docs_reference_links = event.docs_reference_links
-    if (event.docs_reference_titles) entry.docs_reference_titles = event.docs_reference_titles
-    if (fieldsIndex !== undefined) entry.fieldsIndex = fieldsIndex
-
-    const key = JSON.stringify(entry)
-    if (entriesMap.has(key)) return entriesMap.get(key)!
-    const index = entriesPool.length
-    entriesPool.push(entry)
-    entriesMap.set(key, index)
-    return index
-  }
-
-  // Build version index
-  const versionIndex: AuditLogVersionIndex = {}
-  let totalEntries = 0
-
-  for (const [version, pages] of Object.entries(auditLogData)) {
-    versionIndex[version] = {}
-    for (const [page, events] of Object.entries(pages)) {
-      versionIndex[version][page] = events.map((event) => getEntryIndex(event))
-      totalEntries += events.length
-    }
-  }
-
-  // Write shared files
-  const sharedDir = path.join(AUDIT_LOG_DATA_DIR, 'shared')
-  if (!existsSync(sharedDir)) {
-    await mkdirp(sharedDir)
-  }
-
-  await writeFile(path.join(sharedDir, 'entries.json'), JSON.stringify(entriesPool))
-  await writeFile(path.join(sharedDir, 'fields-pool.json'), JSON.stringify(fieldsPool))
-  await writeFile(path.join(AUDIT_LOG_DATA_DIR, 'version-index.json'), JSON.stringify(versionIndex))
-
-  const uniqueEntries = entriesPool.length
-  const uniqueFields = fieldsPool.length
-  const dedupRate = totalEntries > 0 ? ((1 - uniqueEntries / totalEntries) * 100).toFixed(1) : '0'
-  console.log(
-    `✅ Deduplicated audit log data: ${totalEntries} total → ${uniqueEntries} unique entries (${dedupRate}% dedup), ${uniqueFields} unique field lists`,
-  )
+  await writeDeduplicatedAuditLogData(auditLogData)
 }
 
 main()
