Cross-version deduplication for audit logs and GitHub Apps data (#61361)

Co-authored-by: Kevin Heis <heiskr@users.noreply.github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: 3 people

src/audit-logs/data/shared/entries.json

@@ -9,5 +9,5 @@
     "git": "Note: Git events have special access requirements and retention policies that differ from other audit log events. For GitHub Enterprise Cloud, access Git events via the REST API only with 7-day retention. For GitHub Enterprise Server, Git events must be enabled in audit log configuration and are not included in search results.",
     "sso_redirect": "Note: Automatically redirecting users to sign in is currently in beta for Enterprise Managed Users and subject to change."
   },
-  "sha": "961b64fe9080f83cc3f408a8dd1e30fe2eafaee1"
+  "sha": "ad0dc7ebe4a70afe77bb03487060481fe7a9f13a"
 }

src/audit-logs/data/shared/fields-pool.json

@@ -12,6 +12,8 @@ import type {
   RawAuditLogEventT,
   CategoryNotes,
   AuditLogConfig,
+  DeduplicatedAuditLogEntry,
+  AuditLogVersionIndex,
 } from '../types'
 import config from './config.json'
 
@@ -21,6 +23,86 @@ export const AUDIT_LOG_DATA_DIR = 'src/audit-logs/data'
 const auditLogEventsCache = new Map<string, Map<string, AuditLogEventT[]>>()
 const categorizedAuditLogEventsCache = new Map<string, Map<string, CategorizedEvents>>()
 
+// Shared dedup data — loaded once, shared across all versions
+let sharedEntries: DeduplicatedAuditLogEntry[] | null = null
+let sharedFieldsPool: string[][] | null = null
+let sharedVersionIndex: AuditLogVersionIndex | null = null
+let sharedFormatAvailable: boolean | null = null // null = not checked yet
+
+// A missing shared-format file is expected (per-version files are the fallback),
+// but a corrupt or unparseable file should fail loudly rather than silently
+// degrade to the per-version files and hide bad generated data.
+function isFileNotFoundError(err: unknown): boolean {
+  if (!(err instanceof Error) || !('code' in err)) return false
+  const code = (err as NodeJS.ErrnoException).code
+  return code === 'ENOENT' || code === 'ENOTDIR'
+}
+
+function loadSharedFormat(): boolean {
+  if (sharedFormatAvailable !== null) return sharedFormatAvailable
+  try {
+    sharedEntries = readCompressedJsonFileFallback(
+      path.join(AUDIT_LOG_DATA_DIR, 'shared', 'entries.json'),
+    ) as DeduplicatedAuditLogEntry[]
+    sharedFieldsPool = readCompressedJsonFileFallback(
+      path.join(AUDIT_LOG_DATA_DIR, 'shared', 'fields-pool.json'),
+    ) as string[][]
+    sharedVersionIndex = readCompressedJsonFileFallback(
+      path.join(AUDIT_LOG_DATA_DIR, 'version-index.json'),
+    ) as AuditLogVersionIndex
+    // Freeze pool data so reconstructed events (which return references into
+    // these pools) can't be mutated by downstream code and leak across versions.
+    Object.freeze(sharedEntries)
+    Object.freeze(sharedFieldsPool)
+    for (const fields of sharedFieldsPool) Object.freeze(fields)
+    sharedFormatAvailable = true
+  } catch (err) {
+    if (isFileNotFoundError(err)) {
+      // Shared files don't exist — fall back to per-version files silently.
+      sharedFormatAvailable = false
+    } else {
+      // Corrupt JSON, schema mismatch, etc. — surface this instead of hiding it.
+      console.error('Failed to load shared audit log dedup format (corrupt data?):', err)
+      throw err
+    }
+  }
+  return sharedFormatAvailable
+}
+
+function reconstructEventsFromSharedFormat(version: string, page: string): AuditLogEventT[] | null {
+  if (!loadSharedFormat()) return null
+  const indices = sharedVersionIndex?.[version]?.[page]
+  if (!indices) return null
+
+  return indices.map((idx) => {
+    if (idx < 0 || idx >= sharedEntries!.length) {
+      throw new RangeError(
+        `Audit log version-index references entry ${idx} for ${version}/${page}, ` +
+          `but the entries pool only has ${sharedEntries!.length} entries. ` +
+          `The shared dedup data may be stale or corrupt.`,
+      )
+    }
+    const entry = sharedEntries![idx]
+    const event: AuditLogEventT = {
+      action: entry.action,
+      description: entry.description,
+    }
+    if (entry.docs_reference_links) event.docs_reference_links = entry.docs_reference_links
+    if (entry.docs_reference_titles) event.docs_reference_titles = entry.docs_reference_titles
+    if (entry.fieldsIndex !== undefined) {
+      if (entry.fieldsIndex < 0 || entry.fieldsIndex >= sharedFieldsPool!.length) {
+        throw new RangeError(
+          `Audit log entry references fields index ${entry.fieldsIndex} for ${version}/${page}, ` +
+            `but the fields pool only has ${sharedFieldsPool!.length} entries. ` +
+            `The shared dedup data may be stale or corrupt.`,
+        )
+      }
+      event.fields = sharedFieldsPool![entry.fieldsIndex]
+    }
+    return event
+  })
+}
+
 type PipelineConfig = {
   sha: string
   appendedDescriptions: Record<string, string>
@@ -169,21 +251,24 @@ async function resolveReferenceLinksToTitles(
 // ]
 export function getAuditLogEvents(page: string, version: string): AuditLogEventT[] {
   const openApiVersion = getOpenApiVersion(version)
-  const auditLogFileName = path.join(AUDIT_LOG_DATA_DIR, openApiVersion, `${page}.json`)
 
   // If the data isn't cached for an entire version or a particular page, read
-  // the data from the JSON file the first time around
+  // the data from the shared dedup format or fall back to per-version JSON files
   if (!auditLogEventsCache.has(openApiVersion)) {
     auditLogEventsCache.set(openApiVersion, new Map())
-    auditLogEventsCache.get(openApiVersion)?.set(page, [])
-    auditLogEventsCache
-      .get(openApiVersion)
-      ?.set(page, readCompressedJsonFileFallback(auditLogFileName) as AuditLogEventT[])
-  } else if (!auditLogEventsCache.get(openApiVersion)?.has(page)) {
-    auditLogEventsCache.get(openApiVersion)?.set(page, [])
-    auditLogEventsCache
-      .get(openApiVersion)
-      ?.set(page, readCompressedJsonFileFallback(auditLogFileName) as AuditLogEventT[])
+  }
+  if (!auditLogEventsCache.get(openApiVersion)?.has(page)) {
+    // Try shared deduplicated format first
+    const events = reconstructEventsFromSharedFormat(openApiVersion, page)
+    if (events) {
+      auditLogEventsCache.get(openApiVersion)?.set(page, events)
+    } else {
+      // Fall back to per-version JSON file
+      const auditLogFileName = path.join(AUDIT_LOG_DATA_DIR, openApiVersion, `${page}.json`)
+      auditLogEventsCache
+        .get(openApiVersion)
+        ?.set(page, readCompressedJsonFileFallback(auditLogFileName) as AuditLogEventT[])
+    }
   }
 
   const auditLogEvents = auditLogEventsCache.get(openApiVersion)?.get(page)

src/audit-logs/data/version-index.json

@@ -17,7 +17,12 @@ import { getContents, getCommitSha } from '@/workflows/git-utils'
 import { latest, latestStable, releaseCandidate } from '@/versions/lib/enterprise-server-releases'
 import { loadPages, loadPageMap } from '@/frame/lib/page-data'
 import loadRedirects from '@/redirects/lib/precompile'
-import type { AuditLogEventT, VersionedAuditLogData } from '../types'
+import type {
+  AuditLogEventT,
+  VersionedAuditLogData,
+  DeduplicatedAuditLogEntry,
+  AuditLogVersionIndex,
+} from '../types'
 
 if (!process.env.GITHUB_TOKEN) {
   throw new Error('GITHUB_TOKEN environment variable must be set to run this script')
@@ -209,6 +214,78 @@ async function main() {
       }
     }
   }
+
+  // Write deduplicated shared format
+  await writeDeduplicatedFormat(auditLogData)
+}
+
+async function writeDeduplicatedFormat(auditLogData: VersionedAuditLogData) {
+  console.log(`\n▶️  Writing deduplicated audit log data...\n`)
+
+  // Build fields pool: unique fields arrays
+  const fieldsPool: string[][] = []
+  const fieldsMap = new Map<string, number>() // JSON key → index
+
+  function getFieldsIndex(fields: string[] | undefined): number | undefined {
+    if (!fields || fields.length === 0) return undefined
+    const key = JSON.stringify(fields)
+    if (fieldsMap.has(key)) return fieldsMap.get(key)!
+    const index = fieldsPool.length
+    fieldsPool.push(fields)
+    fieldsMap.set(key, index)
+    return index
+  }
+
+  // Build entries pool: unique events (with fields replaced by index)
+  const entriesPool: DeduplicatedAuditLogEntry[] = []
+  const entriesMap = new Map<string, number>() // JSON key → index
+
+  function getEntryIndex(event: AuditLogEventT): number {
+    const fieldsIndex = getFieldsIndex(event.fields)
+    const entry: DeduplicatedAuditLogEntry = {
+      action: event.action,
+      description: event.description,
+    }
+    if (event.docs_reference_links) entry.docs_reference_links = event.docs_reference_links
+    if (event.docs_reference_titles) entry.docs_reference_titles = event.docs_reference_titles
+    if (fieldsIndex !== undefined) entry.fieldsIndex = fieldsIndex
+
+    const key = JSON.stringify(entry)
+    if (entriesMap.has(key)) return entriesMap.get(key)!
+    const index = entriesPool.length
+    entriesPool.push(entry)
+    entriesMap.set(key, index)
+    return index
+  }
+
+  // Build version index
+  const versionIndex: AuditLogVersionIndex = {}
+  let totalEntries = 0
+
+  for (const [version, pages] of Object.entries(auditLogData)) {
+    versionIndex[version] = {}
+    for (const [page, events] of Object.entries(pages)) {
+      versionIndex[version][page] = events.map((event) => getEntryIndex(event))
+      totalEntries += events.length
+    }
+  }
+
+  // Write shared files
+  const sharedDir = path.join(AUDIT_LOG_DATA_DIR, 'shared')
+  if (!existsSync(sharedDir)) {
+    await mkdirp(sharedDir)
+  }
+
+  await writeFile(path.join(sharedDir, 'entries.json'), JSON.stringify(entriesPool))
+  await writeFile(path.join(sharedDir, 'fields-pool.json'), JSON.stringify(fieldsPool))
+  await writeFile(path.join(AUDIT_LOG_DATA_DIR, 'version-index.json'), JSON.stringify(versionIndex))
+
+  const uniqueEntries = entriesPool.length
+  const uniqueFields = fieldsPool.length
+  const dedupRate = totalEntries > 0 ? ((1 - uniqueEntries / totalEntries) * 100).toFixed(1) : '0'
+  console.log(
+    `✅ Deduplicated audit log data: ${totalEntries} total → ${uniqueEntries} unique entries (${dedupRate}% dedup), ${uniqueFields} unique field lists`,
+  )
 }
 
 main()