diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc8e1788..8d4ad36ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Skills/Web: show skill owner avatar + handle on skill cards, lists, and detail pages (#312) (thanks @ianalloway). - Skills/Web: add file viewer for skill version files on detail page (#44) (thanks @regenrek). - CLI: add `uninstall` command for skills (#241) (thanks @superlowburn). +- CI/Security: add TruffleHog pull-request scanning for verified leaked credentials (#505) (thanks @akses0). ### Changed - Quality gate: language-aware word counting (`Intl.Segmenter`) and new `cjkChars` signal to reduce false rejects for non-Latin docs. @@ -17,6 +18,8 @@ - Skills: reserve deleted slugs for prior owners (90-day cooldown) to prevent squatting; add admin reclaim flow (#298) (thanks @autogame-17). - Moderation: ban flow soft-deletes owned skills (reversible) and removes them from vector search (#298) (thanks @autogame-17). - LLM helpers: centralize OpenAI Responses text extraction for changelog/summary/eval flows (#502) (thanks @ianalloway). +- Rate limiting: apply authenticated quotas by user bucket (vs shared IP), emit delay-based reset headers, and improve CLI 429 guidance/retries (#412) (thanks @lc0rp). +- Search/listing performance: cut embedding hydration and badge read bandwidth via `embeddingSkillMap` + denormalized skill badges; shift stat-doc sync to low-frequency cron (#441) (thanks @sethconvex). ### Fixed - Admin API: `POST /api/v1/users/reclaim` now performs non-destructive root-slug owner transfer diff --git a/convex/crons.ts b/convex/crons.ts index f9f8743f6..083d1c64f 100644 --- a/convex/crons.ts +++ b/convex/crons.ts @@ -19,18 +19,30 @@ crons.interval( crons.interval( 'skill-stats-backfill', - { minutes: 10 }, + { hours: 6 }, internal.statsMaintenance.runSkillStatBackfillInternal, { batchSize: 200, maxBatches: 5 }, ) +// Runs frequently to keep dailyStats/trending accurate, +// but does NOT patch skill documents (only writes to skillDailyStats). crons.interval( 'skill-stat-events', - { minutes: 5 }, + { minutes: 15 }, internal.skillStatEvents.processSkillStatEventsAction, {}, ) +// Syncs accumulated stat deltas to skill documents every 6 hours. +// Runs infrequently to avoid thundering-herd reactive query invalidation. +// Uses processedAt field to track progress (independent of the action cursor). +crons.interval( + 'skill-doc-stat-sync', + { hours: 6 }, + internal.skillStatEvents.processSkillStatEventsInternal, + { batchSize: 500 }, +) + crons.interval( 'global-stats-update', { minutes: 60 }, diff --git a/convex/devSeed.ts b/convex/devSeed.ts index 37f731715..11c304879 100644 --- a/convex/devSeed.ts +++ b/convex/devSeed.ts @@ -435,6 +435,7 @@ export const seedSkillMutation = internalMutation({ visibility: 'latest-approved', updatedAt: now, }) + await ctx.db.insert('embeddingSkillMap', { embeddingId, skillId }) await ctx.db.patch(skillId, { latestVersionId: versionId, diff --git a/convex/lib/badges.ts b/convex/lib/badges.ts index 37a899f81..a824aa690 100644 --- a/convex/lib/badges.ts +++ b/convex/lib/badges.ts @@ -35,7 +35,7 @@ export async function getSkillBadgeMap( const records = await ctx.db .query('skillBadges') .withIndex('by_skill', (q) => q.eq('skillId', skillId)) - .collect() + .take(10) return buildBadgeMap(records) } diff --git a/convex/maintenance.test.ts b/convex/maintenance.test.ts index 75dd6545c..8a43d97d3 100644 --- a/convex/maintenance.test.ts +++ b/convex/maintenance.test.ts @@ -37,6 +37,7 @@ const { backfillSkillSummariesInternalHandler, cleanupEmptySkillsInternalHandler, nominateEmptySkillSpammersInternalHandler, + upsertSkillBadgeRecordInternal, } = await import('./maintenance') const { internal } = await import('./_generated/api') const { generateSkillSummary } = await import('./lib/skillSummary') @@ -196,6 +197,81 @@ describe('maintenance backfill', () => { }) }) +describe('maintenance badge denormalization', () => { + it('upserts table badge and keeps skill.badges in sync', async () => { + const unique = vi.fn().mockResolvedValue(null) + const query = vi.fn().mockReturnValue({ + withIndex: () => ({ unique }), + }) + const insert = vi.fn().mockResolvedValue('skillBadges:1') + const get = vi.fn().mockResolvedValue({ _id: 'skills:1', badges: undefined }) + const patch = vi.fn().mockResolvedValue(undefined) + + const ctx = { + db: { + query, + insert, + get, + patch, + }, + } as never + + const result = await (upsertSkillBadgeRecordInternal as { _handler: Function })._handler(ctx, { + skillId: 'skills:1', + kind: 'highlighted', + byUserId: 'users:1', + at: 123, + }) + + expect(result).toEqual({ inserted: true }) + expect(insert).toHaveBeenCalledWith('skillBadges', { + skillId: 'skills:1', + kind: 'highlighted', + byUserId: 'users:1', + at: 123, + }) + expect(patch).toHaveBeenCalledWith('skills:1', { + badges: { + highlighted: { byUserId: 'users:1', at: 123 }, + }, + }) + }) + + it('resyncs denormalized badge even when table record already exists', async () => { + const unique = vi.fn().mockResolvedValue({ _id: 'skillBadges:existing' }) + const query = vi.fn().mockReturnValue({ + withIndex: () => ({ unique }), + }) + const insert = vi.fn() + const get = vi.fn().mockResolvedValue({ _id: 'skills:1', badges: {} }) + const patch = vi.fn().mockResolvedValue(undefined) + + const ctx = { + db: { + query, + insert, + get, + patch, + }, + } as never + + const result = await (upsertSkillBadgeRecordInternal as { _handler: Function })._handler(ctx, { + skillId: 'skills:1', + kind: 'official', + byUserId: 'users:2', + at: 456, + }) + + expect(result).toEqual({ inserted: false }) + expect(insert).not.toHaveBeenCalled() + expect(patch).toHaveBeenCalledWith('skills:1', { + badges: { + official: { byUserId: 'users:2', at: 456 }, + }, + }) + }) +}) + describe('maintenance fingerprint backfill', () => { it('backfills fingerprint field and inserts index entry', async () => { const { hashSkillFiles } = await import('./lib/skills') diff --git a/convex/maintenance.ts b/convex/maintenance.ts index aff67bb68..fb9c91158 100644 --- a/convex/maintenance.ts +++ b/convex/maintenance.ts @@ -642,17 +642,32 @@ export const upsertSkillBadgeRecordInternal = internalMutation({ at: v.number(), }, handler: async (ctx, args) => { + const syncDenormalizedBadge = async () => { + const skill = await ctx.db.get(args.skillId) + if (!skill) return + await ctx.db.patch(args.skillId, { + badges: { + ...(skill.badges as Record | undefined), + [args.kind]: { byUserId: args.byUserId, at: args.at }, + }, + }) + } + const existing = await ctx.db .query('skillBadges') .withIndex('by_skill_kind', (q) => q.eq('skillId', args.skillId).eq('kind', args.kind)) .unique() - if (existing) return { inserted: false as const } + if (existing) { + await syncDenormalizedBadge() + return { inserted: false as const } + } await ctx.db.insert('skillBadges', { skillId: args.skillId, kind: args.kind, byUserId: args.byUserId, at: args.at, }) + await syncDenormalizedBadge() return { inserted: true as const } }, }) @@ -1411,6 +1426,109 @@ export const nominateEmptySkillSpammers: ReturnType = action({ }, }) +// Backfill embeddingSkillMap from existing skillEmbeddings. +// Run once after deploying the schema change: +// npx convex run maintenance:backfillEmbeddingSkillMapInternal --prod +export const backfillEmbeddingSkillMapInternal = internalMutation({ + args: { + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + }, + handler: async (ctx, args) => { + const batchSize = clampInt(args.batchSize ?? 200, 10, 500) + const { page, continueCursor, isDone } = await ctx.db + .query('skillEmbeddings') + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }) + + let inserted = 0 + for (const embedding of page) { + const existing = await ctx.db + .query('embeddingSkillMap') + .withIndex('by_embedding', (q) => q.eq('embeddingId', embedding._id)) + .unique() + if (!existing) { + await ctx.db.insert('embeddingSkillMap', { + embeddingId: embedding._id, + skillId: embedding.skillId, + }) + inserted++ + } + } + + if (!isDone) { + await ctx.scheduler.runAfter(0, internal.maintenance.backfillEmbeddingSkillMapInternal, { + cursor: continueCursor, + batchSize: args.batchSize, + }) + } + + return { inserted, isDone, scanned: page.length } + }, +}) + +// Sync skillBadges table → denormalized skill.badges field. +// Run after deploying the badge-read removal to ensure all skills +// have up-to-date badges on the skill doc itself. +export const backfillDenormalizedBadgesInternal = internalMutation({ + args: { + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + }, + handler: async (ctx, args) => { + const batchSize = clampInt(args.batchSize ?? 100, 10, 200) + const { page, continueCursor, isDone } = await ctx.db + .query('skills') + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }) + + let patched = 0 + for (const skill of page) { + const records = await ctx.db + .query('skillBadges') + .withIndex('by_skill', (q) => q.eq('skillId', skill._id)) + .take(10) + + // Build canonical badge map from the table + const canonical: Record; at: number }> = {} + for (const r of records) { + canonical[r.kind] = { byUserId: r.byUserId, at: r.at } + } + + // Compare with existing denormalized badges (keys + values) + const existing = (skill.badges ?? {}) as Record< + string, + { byUserId?: Id<'users'>; at?: number } | undefined + > + const canonicalKeys = Object.keys(canonical) + const existingKeys = Object.keys(existing).filter((k) => existing[k] !== undefined) + const needsPatch = + canonicalKeys.length !== existingKeys.length || + canonicalKeys.some((k) => { + const current = existing[k] + const next = canonical[k] + return ( + !current || + current.byUserId !== next.byUserId || + current.at !== next.at + ) + }) + + if (needsPatch) { + await ctx.db.patch(skill._id, { badges: canonical }) + patched++ + } + } + + if (!isDone) { + await ctx.scheduler.runAfter(0, internal.maintenance.backfillDenormalizedBadgesInternal, { + cursor: continueCursor, + batchSize: args.batchSize, + }) + } + + return { patched, isDone, scanned: page.length } + }, +}) + function clampInt(value: number, min: number, max: number) { const rounded = Math.trunc(value) if (!Number.isFinite(rounded)) return min diff --git a/convex/schema.ts b/convex/schema.ts index a8993da53..dc2ff4a69 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -310,6 +310,14 @@ const skillEmbeddings = defineTable({ filterFields: ['visibility'], }) +// Lightweight lookup: embeddingId → skillId (~100 bytes per doc). +// Avoids reading full skillEmbeddings docs (~12KB each with vector) +// during search hydration. +const embeddingSkillMap = defineTable({ + embeddingId: v.id('skillEmbeddings'), + skillId: v.id('skills'), +}).index('by_embedding', ['embeddingId']) + const skillDailyStats = defineTable({ skillId: v.id('skills'), day: v.number(), @@ -576,6 +584,7 @@ export default defineSchema({ skillBadges, soulVersionFingerprints, skillEmbeddings, + embeddingSkillMap, soulEmbeddings, skillDailyStats, skillLeaderboards, diff --git a/convex/search.test.ts b/convex/search.test.ts index 2ee69cd13..dd04e44ac 100644 --- a/convex/search.test.ts +++ b/convex/search.test.ts @@ -4,9 +4,8 @@ import { describe, expect, it, vi } from 'vitest' import { tokenize } from './lib/searchText' import { __test, hydrateResults, lexicalFallbackSkills, searchSkills } from './search' -const { generateEmbeddingMock, getSkillBadgeMapsMock } = vi.hoisted(() => ({ +const { generateEmbeddingMock } = vi.hoisted(() => ({ generateEmbeddingMock: vi.fn(), - getSkillBadgeMapsMock: vi.fn(), })) vi.mock('./lib/embeddings', () => ({ @@ -14,7 +13,6 @@ vi.mock('./lib/embeddings', () => ({ })) vi.mock('./lib/badges', () => ({ - getSkillBadgeMaps: getSkillBadgeMapsMock, isSkillHighlighted: (skill: { badges?: Record }) => Boolean(skill.badges?.highlighted), })) @@ -50,9 +48,8 @@ describe('search helpers', () => { ] const runQuery = vi .fn() - .mockResolvedValueOnce([]) - .mockResolvedValueOnce([]) - .mockResolvedValueOnce(fallback) + .mockResolvedValueOnce([]) // hydrateResults + .mockResolvedValueOnce(fallback) // lexicalFallbackSkills const result = await searchSkillsHandler( { @@ -71,18 +68,15 @@ describe('search helpers', () => { }) it('applies highlightedOnly filtering in lexical fallback', async () => { - const highlighted = makeSkillDoc({ - id: 'skills:hl', - slug: 'orf-highlighted', - displayName: 'ORF Highlighted', - }) + const highlighted = { + ...makeSkillDoc({ + id: 'skills:hl', + slug: 'orf-highlighted', + displayName: 'ORF Highlighted', + }), + badges: { highlighted: { byUserId: 'users:mod', at: 1 } }, + } const plain = makeSkillDoc({ id: 'skills:plain', slug: 'orf-plain', displayName: 'ORF Plain' }) - getSkillBadgeMapsMock.mockResolvedValueOnce( - new Map([ - ['skills:hl', { highlighted: { byUserId: 'users:mod', at: 1 } }], - ['skills:plain', {}], - ]), - ) const result = await lexicalFallbackSkillsHandler( makeLexicalCtx({ @@ -104,12 +98,6 @@ describe('search helpers', () => { moderationFlags: ['flagged.suspicious'], }) const clean = makeSkillDoc({ id: 'skills:clean', slug: 'orf-clean', displayName: 'ORF Clean' }) - getSkillBadgeMapsMock.mockResolvedValueOnce( - new Map([ - ['skills:suspicious', {}], - ['skills:clean', {}], - ]), - ) const result = await lexicalFallbackSkillsHandler( makeLexicalCtx({ @@ -125,7 +113,6 @@ describe('search helpers', () => { it('includes exact slug match from by_slug even when recent scan is empty', async () => { const exactSlugSkill = makeSkillDoc({ id: 'skills:orf', slug: 'orf', displayName: 'ORF' }) - getSkillBadgeMapsMock.mockResolvedValueOnce(new Map([['skills:orf', {}]])) const ctx = makeLexicalCtx({ exactSlugSkill, recentSkills: [], @@ -197,9 +184,8 @@ describe('search helpers', () => { const runQuery = vi .fn() - .mockResolvedValueOnce(vectorEntries) - .mockResolvedValueOnce([]) - .mockResolvedValueOnce(fallbackEntries) + .mockResolvedValueOnce(vectorEntries) // hydrateResults + .mockResolvedValueOnce(fallbackEntries) // lexicalFallbackSkills const result = await searchSkillsHandler( { @@ -237,6 +223,9 @@ describe('search helpers', () => { if (id === 'skillVersions:1') return { _id: 'skillVersions:1', version: '1.0.0' } return null }), + query: vi.fn(() => ({ + withIndex: () => ({ unique: vi.fn().mockResolvedValue(null) }), + })), }, }, { embeddingIds: ['skillEmbeddings:1'], nonSuspiciousOnly: true }, diff --git a/convex/search.ts b/convex/search.ts index a7a23a016..9c0bb0e04 100644 --- a/convex/search.ts +++ b/convex/search.ts @@ -3,7 +3,7 @@ import { internal } from './_generated/api' import type { Doc, Id } from './_generated/dataModel' import type { QueryCtx } from './_generated/server' import { action, internalQuery } from './_generated/server' -import { getSkillBadgeMaps, isSkillHighlighted, type SkillBadgeMap } from './lib/badges' +import { isSkillHighlighted } from './lib/badges' import { generateEmbedding } from './lib/embeddings' import { toPublicSkill, toPublicSoul, toPublicUser } from './lib/public' import { matchesExactTokens, tokenize } from './lib/searchText' @@ -40,7 +40,7 @@ const SLUG_PREFIX_BOOST = 0.8 const NAME_EXACT_BOOST = 1.1 const NAME_PREFIX_BOOST = 0.6 const POPULARITY_WEIGHT = 0.08 -const FALLBACK_SCAN_LIMIT = 1200 +const FALLBACK_SCAN_LIMIT = 500 function getNextCandidateLimit(current: number, max: number) { const next = Math.min(current * 2, max) @@ -149,21 +149,11 @@ export const searchSkills: ReturnType = action({ results.map((result) => [result._id, result._score]), ) - const badgeMapEntries = (await ctx.runQuery(internal.search.getSkillBadgeMapsInternal, { - skillIds: hydrated.map((entry) => entry.skill._id), - })) as Array<[Id<'skills'>, SkillBadgeMap]> - const badgeMapBySkillId = new Map(badgeMapEntries) - const hydratedWithBadges = hydrated.map((entry) => ({ - ...entry, - skill: { - ...entry.skill, - badges: badgeMapBySkillId.get(entry.skill._id) ?? {}, - }, - })) - + // Skills already have badges from their docs (via toPublicSkill). + // No need for a separate badge table lookup. const filtered = args.highlightedOnly - ? hydratedWithBadges.filter((entry) => isSkillHighlighted(entry.skill)) - : hydratedWithBadges + ? hydrated.filter((entry) => isSkillHighlighted(entry.skill)) + : hydrated exactMatches = filtered.filter((entry) => matchesExactTokens(queryTokens, [ @@ -215,14 +205,6 @@ export const searchSkills: ReturnType = action({ }, }) -export const getBadgeMapsForSkills = internalQuery({ - args: { skillIds: v.array(v.id('skills')) }, - handler: async (ctx, args): Promise, SkillBadgeMap]>> => { - const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds) - return Array.from(badgeMap.entries()) - }, -}) - export const hydrateResults = internalQuery({ args: { embeddingIds: v.array(v.id('skillEmbeddings')), @@ -233,21 +215,26 @@ export const hydrateResults = internalQuery({ const entries: Array = await Promise.all( args.embeddingIds.map(async (embeddingId) => { - const embedding = await ctx.db.get(embeddingId) - if (!embedding) return null - const skill = await ctx.db.get(embedding.skillId) + // Use lightweight lookup table (~100 bytes) instead of full embedding doc (~12KB). + const lookup = await ctx.db + .query('embeddingSkillMap') + .withIndex('by_embedding', (q) => q.eq('embeddingId', embeddingId)) + .unique() + // Fallback to full embedding doc for rows not yet backfilled. + const skillId = lookup + ? lookup.skillId + : await ctx.db.get(embeddingId).then((e) => e?.skillId) + if (!skillId) return null + const skill = await ctx.db.get(skillId) if (!skill || skill.softDeletedAt) return null if (args.nonSuspiciousOnly && isSkillSuspicious(skill)) return null - const [version, ownerInfo] = await Promise.all([ - ctx.db.get(embedding.versionId), - getOwnerInfo(skill.ownerUserId), - ]) + const ownerInfo = await getOwnerInfo(skill.ownerUserId) const publicSkill = toPublicSkill(skill) if (!publicSkill) return null return { embeddingId, skill: publicSkill, - version, + version: null as Doc<'skillVersions'> | null, ownerHandle: ownerInfo.handle, owner: ownerInfo.owner, } @@ -309,15 +296,12 @@ export const lexicalFallbackSkills = internalQuery({ const entries = await Promise.all( matched.map(async (skill) => { - const [version, ownerInfo] = await Promise.all([ - skill.latestVersionId ? ctx.db.get(skill.latestVersionId) : Promise.resolve(null), - getOwnerInfo(skill.ownerUserId), - ]) + const ownerInfo = await getOwnerInfo(skill.ownerUserId) const publicSkill = toPublicSkill(skill) if (!publicSkill) return null return { skill: publicSkill, - version, + version: null as Doc<'skillVersions'> | null, ownerHandle: ownerInfo.handle, owner: ownerInfo.owner, } @@ -326,21 +310,11 @@ export const lexicalFallbackSkills = internalQuery({ const validEntries = entries.filter((entry): entry is SkillSearchEntry => entry !== null) if (validEntries.length === 0) return [] - const badgeMap = await getSkillBadgeMaps( - ctx, - validEntries.map((entry) => entry.skill._id), - ) - const withBadges = validEntries.map((entry) => ({ - ...entry, - skill: { - ...entry.skill, - badges: badgeMap.get(entry.skill._id) ?? {}, - }, - })) - + // Skills already have badges from their docs (via toPublicSkill). + // No need for a separate badge table lookup. const filtered = args.highlightedOnly - ? withBadges.filter((entry) => isSkillHighlighted(entry.skill)) - : withBadges + ? validEntries.filter((entry) => isSkillHighlighted(entry.skill)) + : validEntries return filtered.slice(0, limit) }, }) @@ -440,14 +414,6 @@ export const hydrateSoulResults = internalQuery({ }, }) -export const getSkillBadgeMapsInternal = internalQuery({ - args: { skillIds: v.array(v.id('skills')) }, - handler: async (ctx, args) => { - const badgeMap = await getSkillBadgeMaps(ctx, args.skillIds) - return Array.from(badgeMap.entries()) - }, -}) - export const __test = { getNextCandidateLimit, matchesAllTokens, diff --git a/convex/skillStatEvents.ts b/convex/skillStatEvents.ts index d88e8c2d5..f88cd602d 100644 --- a/convex/skillStatEvents.ts +++ b/convex/skillStatEvents.ts @@ -3,14 +3,18 @@ * * Instead of updating skill stats synchronously in the hot path (which can cause * contention when multiple users download/star/install the same skill), we insert - * lightweight event records and process them in batches via a cron job. + * lightweight event records and process them in batches via cron jobs. * - * Flow: - * 1. User action (download, star, install) → insertStatEvent() writes to skillStatEvents table - * 2. Cron job runs every 5 minutes → processSkillStatEventsInternal() processes batches - * 3. Events are aggregated per-skill to minimize database operations - * 4. Stats are applied to skill documents and daily stats tables - * 5. Events are marked as processed (kept forever for auditing) + * Two processing paths run at different frequencies to balance freshness vs bandwidth: + * + * 1. **Daily stats (15-minute cron)** — `processSkillStatEventsAction` + * Writes to skillDailyStats for trending/leaderboards. Uses a cursor in + * skillStatUpdateCursors. Does NOT touch skill documents. + * + * 2. **Skill doc sync (6-hour cron)** — `processSkillStatEventsInternal` + * Patches skill documents with accumulated stat deltas. Uses processedAt + * field to track progress. Runs infrequently because patching skill docs + * invalidates reactive queries for all subscribers (thundering herd). */ import { v } from 'convex/values' @@ -175,7 +179,7 @@ function aggregateEvents(events: Doc<'skillStatEvents'>[]): AggregatedDeltas { /** * Process a batch of unprocessed stat events. * - * Called by cron every 5 minutes. Processes up to batchSize events (default 100). + * Called by the 6-hour cron to sync stats to skill docs. Processes up to batchSize events (default 500). * If the batch is full, schedules an immediate follow-up run to drain the queue. * * Processing steps: @@ -198,7 +202,7 @@ function aggregateEvents(events: Doc<'skillStatEvents'>[]): AggregatedDeltas { export const processSkillStatEventsInternal = internalMutation({ args: { batchSize: v.optional(v.number()) }, handler: async (ctx, args) => { - const batchSize = args.batchSize ?? 100 + const batchSize = args.batchSize ?? 500 const now = Date.now() // Level 1: Fetch a batch of unprocessed events @@ -252,25 +256,13 @@ export const processSkillStatEventsInternal = internalMutation({ installsAllTime: deltas.installsAllTime, installsCurrent: deltas.installsCurrent, }) - await ctx.db.patch(skill._id, { - ...patch, - updatedAt: now, - }) + // Don't update `updatedAt` — stat changes shouldn't move the + // skill's position in the by_active_updated index. + await ctx.db.patch(skill._id, patch) } - // Update daily stats for trending/leaderboards - // We use the ORIGINAL event timestamp (occurredAt) so that: - // - A download at Mon 11:55 PM counts toward Monday's stats - // - Even if the cron processes it on Tuesday - // - // Level 4: bumpDailySkillStats does its own coalescing - multiple - // events on the same day will update the same daily record - for (const occurredAt of deltas.downloadEvents) { - await bumpDailySkillStats(ctx, { skillId, now: occurredAt, downloads: 1 }) - } - for (const occurredAt of deltas.installNewEvents) { - await bumpDailySkillStats(ctx, { skillId, now: occurredAt, installs: 1 }) - } + // NOTE: Daily stats (skillDailyStats) are written by the 15-minute + // action cron (processSkillStatEventsAction), not here. // Mark all events for this skill as processed for (const event of skillEvents) { @@ -352,11 +344,11 @@ const skillDeltaValidator = v.object({ }) /** - * Apply aggregated stats to skills and update the cursor. + * Write aggregated daily stats and advance the cursor. * This is a single atomic mutation that: - * 1. Updates all affected skills with their aggregated deltas - * 2. Updates daily stats for trending - * 3. Advances the cursor to the new position + * 1. Updates daily stats for trending/leaderboards (skillDailyStats) + * 2. Advances the cursor to the new position + * NOTE: Does NOT patch skill documents — that's handled by processSkillStatEventsInternal. */ export const applyAggregatedStatsAndUpdateCursor = internalMutation({ args: { @@ -366,37 +358,8 @@ export const applyAggregatedStatsAndUpdateCursor = internalMutation({ handler: async (ctx, args) => { const now = Date.now() - // Process each skill's aggregated deltas + // Update daily stats for trending/leaderboards for (const delta of args.skillDeltas) { - const skill = await ctx.db.get(delta.skillId) - - // Skill was deleted - skip - if (!skill) { - continue - } - - // Apply aggregated deltas to skill stats - if ( - delta.downloads !== 0 || - delta.stars !== 0 || - delta.comments !== 0 || - delta.installsAllTime !== 0 || - delta.installsCurrent !== 0 - ) { - const patch = applySkillStatDeltas(skill, { - downloads: delta.downloads, - stars: delta.stars, - comments: delta.comments, - installsAllTime: delta.installsAllTime, - installsCurrent: delta.installsCurrent, - }) - await ctx.db.patch(skill._id, { - ...patch, - updatedAt: now, - }) - } - - // Update daily stats for trending/leaderboards for (const occurredAt of delta.downloadEvents) { await bumpDailySkillStats(ctx, { skillId: delta.skillId, now: occurredAt, downloads: 1 }) } diff --git a/convex/skills.ts b/convex/skills.ts index ac8c153dd..9fc7a5470 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -17,7 +17,6 @@ import { getSkillBadgeMap, getSkillBadgeMaps, isSkillHighlighted, - type SkillBadgeMap, } from './lib/badges' import { generateChangelogPreview as buildChangelogPreview } from './lib/changelog' import { @@ -61,7 +60,6 @@ const MAX_LIST_LIMIT = 50 const MAX_PUBLIC_LIST_LIMIT = 200 const MAX_LIST_BULK_LIMIT = 200 const MAX_LIST_TAKE = 1000 -const MAX_BADGE_LOOKUP_SKILLS = 200 const HARD_DELETE_BATCH_SIZE = 100 const HARD_DELETE_VERSION_BATCH_SIZE = 10 const HARD_DELETE_LEADERBOARD_BATCH_SIZE = 25 @@ -510,18 +508,16 @@ type ManagementSkillEntry = { type BadgeKind = Doc<'skillBadges'>['kind'] -async function buildPublicSkillEntries(ctx: QueryCtx, skills: Doc<'skills'>[]) { +async function buildPublicSkillEntries( + ctx: QueryCtx, + skills: Doc<'skills'>[], + opts?: { includeVersion?: boolean }, +) { + const includeVersion = opts?.includeVersion ?? true const ownerInfoCache = new Map< Id<'users'>, Promise<{ ownerHandle: string | null; owner: ReturnType | null }> >() - const badgeMapBySkillId: Map, SkillBadgeMap> = skills.length <= - MAX_BADGE_LOOKUP_SKILLS - ? await getSkillBadgeMaps( - ctx, - skills.map((skill) => skill._id), - ) - : new Map() const getOwnerInfo = (ownerUserId: Id<'users'>) => { const cached = ownerInfoCache.get(ownerUserId) @@ -542,11 +538,10 @@ async function buildPublicSkillEntries(ctx: QueryCtx, skills: Doc<'skills'>[]) { const entries = await Promise.all( skills.map(async (skill) => { const [latestVersionDoc, ownerInfo] = await Promise.all([ - skill.latestVersionId ? ctx.db.get(skill.latestVersionId) : null, + includeVersion && skill.latestVersionId ? ctx.db.get(skill.latestVersionId) : null, getOwnerInfo(skill.ownerUserId), ]) - const badges = badgeMapBySkillId.get(skill._id) ?? {} - const publicSkill = toPublicSkill({ ...skill, badges }) + const publicSkill = toPublicSkill(skill) if (!publicSkill) return null const latestVersion = toPublicSkillListVersion(latestVersionDoc) return { @@ -645,14 +640,24 @@ async function upsertSkillBadge( .unique() if (existing) { await ctx.db.patch(existing._id, { byUserId: userId, at }) - return existing._id + } else { + await ctx.db.insert('skillBadges', { + skillId, + kind, + byUserId: userId, + at, + }) + } + // Keep denormalized badges field on skill doc in sync + const skill = await ctx.db.get(skillId) + if (skill) { + await ctx.db.patch(skillId, { + badges: { + ...(skill.badges as Record | undefined), + [kind]: { byUserId: userId, at }, + }, + }) } - return ctx.db.insert('skillBadges', { - skillId, - kind, - byUserId: userId, - at, - }) } async function removeSkillBadge(ctx: MutationCtx, skillId: Id<'skills'>, kind: BadgeKind) { @@ -663,6 +668,12 @@ async function removeSkillBadge(ctx: MutationCtx, skillId: Id<'skills'>, kind: B if (existing) { await ctx.db.delete(existing._id) } + // Keep denormalized badges field on skill doc in sync + const skill = await ctx.db.get(skillId) + if (skill) { + const { [kind]: _, ...remainingBadges } = (skill.badges ?? {}) as Record + await ctx.db.patch(skillId, { badges: remainingBadges }) + } } export const getBySlug = query({ @@ -1614,8 +1625,9 @@ export const listPublicPageV2 = query({ }) : result.page - // Build the public skill entries (fetch latestVersion + ownerHandle) - const items = await buildPublicSkillEntries(ctx, filteredPage) + // Build the public skill entries — skip version doc reads to reduce bandwidth. + // Version data is only needed for detail pages, not the listing. + const items = await buildPublicSkillEntries(ctx, filteredPage, { includeVersion: false }) return { ...result, page: items } }, }) @@ -3773,6 +3785,8 @@ export const insertVersion = internalMutation({ visibility: embeddingVisibilityFor(true, isApproved), updatedAt: now, }) + // Lightweight lookup so search hydration can skip reading the 12KB embedding doc + await ctx.db.insert('embeddingSkillMap', { embeddingId, skillId: skill._id }) if (latestBefore) { const previousEmbedding = await ctx.db diff --git a/src/routes/skills/-SkillsResults.tsx b/src/routes/skills/-SkillsResults.tsx index 521722b3c..f454dad31 100644 --- a/src/routes/skills/-SkillsResults.tsx +++ b/src/routes/skills/-SkillsResults.tsx @@ -47,7 +47,6 @@ export function SkillsResults({
{sorted.map((entry) => { const skill = entry.skill - const isPlugin = Boolean(entry.latestVersion?.parsed?.clawdis?.nix?.plugin) const ownerHandle = entry.owner?.handle ?? entry.owner?.name ?? entry.ownerHandle ?? null const skillHref = buildSkillHref(skill, ownerHandle) return ( @@ -56,7 +55,6 @@ export function SkillsResults({ skill={skill} href={skillHref} badge={getSkillBadges(skill)} - chip={isPlugin ? 'Plugin bundle (nix)' : undefined} summaryFallback="Agent-ready skill pack." meta={
@@ -74,7 +72,6 @@ export function SkillsResults({
{sorted.map((entry) => { const skill = entry.skill - const isPlugin = Boolean(entry.latestVersion?.parsed?.clawdis?.nix?.plugin) const ownerHandle = entry.owner?.handle ?? entry.owner?.name ?? entry.ownerHandle ?? null const skillHref = buildSkillHref(skill, ownerHandle) return ( @@ -88,15 +85,11 @@ export function SkillsResults({ {badge} ))} - {isPlugin ? Plugin bundle (nix) : null}
{skill.summary ?? 'No summary provided.'}
- {isPlugin ? ( -
Bundle includes SKILL.md, CLI, and config.
- ) : null}