diff --git a/.github/workflows/pr-code-quality-commenter.yml b/.github/workflows/pr-code-quality-commenter.yml new file mode 100644 index 00000000..f29ab6cd --- /dev/null +++ b/.github/workflows/pr-code-quality-commenter.yml @@ -0,0 +1,70 @@ +name: PR Code Quality Commenter + +on: + workflow_run: + workflows: ["PR Code Quality Checks"] + types: + - completed + +permissions: + pull-requests: write + issues: write + +jobs: + comment: + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: pr-comments + continue-on-error: true + + - name: Post Comments + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // Find the PR associated with this run + const pulls = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: context.payload.workflow_run.head_sha + }); + const pr = pulls.data[0]; + if (!pr) { + console.log("No pull request found for commit: " + context.payload.workflow_run.head_sha); + return; + } + const prNumber = pr.number; + console.log(`Associated PR: #${prNumber}`); + + const postComment = async (dirName, fileName) => { + const dirPath = path.join(process.env.GITHUB_WORKSPACE, 'pr-comments', dirName); + if (!fs.existsSync(dirPath)) { + console.log(`Directory ${dirName} does not exist. Skipping.`); + return; + } + + const commentFile = path.join(dirPath, fileName); + if (fs.existsSync(commentFile)) { + const body = fs.readFileSync(commentFile, 'utf8').trim(); + if (body) { + console.log(`Posting comment from ${dirName} to PR #${prNumber}...`); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + } + } + }; + + await postComment('security-warning', 'security-warning.txt'); + await postComment('quality-gate-warning', 'quality-gate-warning.txt'); diff --git a/.github/workflows/pr-code-quality.yml b/.github/workflows/pr-code-quality.yml new file mode 100644 index 00000000..cf969122 --- /dev/null +++ b/.github/workflows/pr-code-quality.yml @@ -0,0 +1,261 @@ +name: PR Code Quality Checks + +on: + pull_request: + branches: [main, master] + types: [opened, synchronize, reopened] + +permissions: {} + +jobs: + # ───────────────────────────────────────── + # 1. TypeScript type checking + # ───────────────────────────────────────── + typecheck: + name: TypeScript Check + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + cache: "npm" + + - name: Install dependencies + run: npm ci + + - name: Run TypeScript compiler check + run: npx tsc --noEmit + continue-on-error: true + + # ───────────────────────────────────────── + # 2. ESLint - catch code issues + # ───────────────────────────────────────── + lint: + name: ESLint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + cache: "npm" + + - name: Install dependencies + run: npm ci + + - name: Run ESLint + run: npx eslint . --ext .ts,.tsx --max-warnings=20 + continue-on-error: true + + # ───────────────────────────────────────── + # 3. Build check — does it compile? + # ───────────────────────────────────────── + build: + name: Build Check + runs-on: ubuntu-latest + permissions: + contents: read + env: + # Dummy env vars so Next.js build doesn't fail on missing secrets + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${{ secrets.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY || 'pk_test_placeholder' }} + CLERK_SECRET_KEY: ${{ secrets.CLERK_SECRET_KEY || 'sk_test_placeholder' }} + DATABASE_URL: ${{ secrets.DATABASE_URL || 'postgresql://placeholder' }} + NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'https://placeholder.supabase.co' }} + NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'placeholder' }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + cache: "npm" + + - name: Install dependencies + run: npm ci + + - name: Build Next.js app + run: npm run build + continue-on-error: true + + # ───────────────────────────────────────── + # 4. Security scan — detect common issues + # ───────────────────────────────────────── + security-scan: + name: Security Scan + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + return files.map(f => f.filename); + + - name: Scan for hardcoded secrets patterns + id: secret-scan + run: | + echo "Scanning for potential secrets in changed files..." + ISSUES="" + + # Check for common secret patterns + if git diff origin/main...HEAD -- '*.ts' '*.tsx' '*.js' | grep -E "(password|secret|api_key|apikey|token)\s*=\s*['\"][^'\"]{8,}" --include="*.ts" --include="*.tsx" -i; then + ISSUES="$ISSUES\n- Possible hardcoded secret detected" + fi + + # Check for NEXT_PUBLIC_ on sensitive vars + if git diff origin/main...HEAD | grep -E "NEXT_PUBLIC_(DATABASE_URL|SECRET|PASSWORD|PRIVATE)" -i; then + ISSUES="$ISSUES\n- Sensitive variable exposed via NEXT_PUBLIC_ prefix" + fi + + # Check for console.log with sensitive data patterns + if git diff origin/main...HEAD -- '*.ts' '*.tsx' | grep -E "console\.(log|error)\(.*?(password|token|secret|key)" -i; then + ISSUES="$ISSUES\n- Possible sensitive data in console.log" + fi + + if [ -n "$ISSUES" ]; then + echo "security_issues=true" >> $GITHUB_OUTPUT + echo "issues=$ISSUES" >> $GITHUB_OUTPUT + else + echo "security_issues=false" >> $GITHUB_OUTPUT + fi + continue-on-error: true + + - name: Save security findings comment + if: steps.secret-scan.outputs.security_issues == 'true' + uses: actions/github-script@v6 + env: + SECURITY_ISSUES: ${{ steps.secret-scan.outputs.issues }} + with: + script: | + const fs = require('fs'); + const path = require('path'); + const dir = './security-warning'; + if (!fs.existsSync(dir)){ + fs.mkdirSync(dir, { recursive: true }); + } + const body = `## Security Scan Warning\n\nPotential security issues detected in this PR:\n${process.env.SECURITY_ISSUES}\n\nPlease review before merging. @knoxiboy`; + fs.writeFileSync(path.join(dir, 'security-warning.txt'), body); + + + - name: Upload security findings artifact + if: steps.secret-scan.outputs.security_issues == 'true' + uses: actions/upload-artifact@v4 + with: + name: security-warning + path: security-warning/ + if-no-files-found: ignore + + # ───────────────────────────────────────── + # 5. PR quality gate — basic checks + # ───────────────────────────────────────── + pr-quality-gate: + name: PR Quality Gate + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check PR quality + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const pr = context.payload.pull_request; + const title = pr.title || ''; + const body = pr.body || ''; + const issues = []; + + // 1. Title format check + const validPrefixes = ['feat', 'fix', 'docs', 'style', 'refactor', 'test', 'chore', 'perf', 'ci', 'build', 'revert']; + const hasValidPrefix = validPrefixes.some(p => title.toLowerCase().startsWith(p)); + if (!hasValidPrefix) { + issues.push(`**Title format**: Title should start with a conventional commit prefix (e.g. \`feat:\`, \`fix:\`, \`docs:\`). Current: \`${title}\``); + } + + // 2. PR description check + if (body.trim().length < 50) { + issues.push('**Description**: PR description is too short. Please describe what changes were made and why.'); + } + + // 3. Check for linked issue + const hasLinkedIssue = /closes?\s+#\d+|fixes?\s+#\d+|resolves?\s+#\d+/i.test(body); + if (!hasLinkedIssue) { + issues.push('**Linked Issue**: No linked issue found. Please add `Closes #` to your PR description.'); + } + + // 4. Check for test mention + const hasTestMention = /test|spec|jest|vitest|playwright/i.test(body); + if (!hasTestMention) { + issues.push('**Testing**: No mention of tests in the PR description. Please describe how you tested your changes.'); + } + + // Post results + if (issues.length > 0) { + const comment = `## PR Quality Check\n\nThe following items need attention:\n\n${issues.map(i => `- ${i}`).join('\n')}\n\n> These are suggestions to improve PR quality. The PR can still be merged after review.`; + const fs = require('fs'); + const path = require('path'); + const dir = './quality-gate-warning'; + if (!fs.existsSync(dir)){ + fs.mkdirSync(dir, { recursive: true }); + } + fs.writeFileSync(path.join(dir, 'quality-gate-warning.txt'), comment); + console.log('Saved quality gate feedback to file'); + } else { + console.log('PR passed all quality checks'); + } + + - name: Upload quality gate warning artifact + uses: actions/upload-artifact@v4 + with: + name: quality-gate-warning + path: quality-gate-warning/ + if-no-files-found: ignore + + # ───────────────────────────────────────── + # 6. Dependency audit + # ───────────────────────────────────────── + dependency-audit: + name: Dependency Audit + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + cache: "npm" + + - name: Check for new dependencies added + id: dep-check + run: | + git diff origin/main...HEAD -- package.json | grep "^+" | grep -v "^+++" | grep -E '"[^"]+"\s*:\s*"' || echo "No new dependencies" + + - name: Run npm audit on high severity + run: npm audit --audit-level=high + continue-on-error: true diff --git a/drizzle/0009_add_doubt_embeddings_pgvector.sql b/drizzle/0009_add_doubt_embeddings_pgvector.sql new file mode 100644 index 00000000..955ea510 --- /dev/null +++ b/drizzle/0009_add_doubt_embeddings_pgvector.sql @@ -0,0 +1,45 @@ +-- Enable pgvector for embedding storage and similarity search +CREATE EXTENSION IF NOT EXISTS vector; + +--> statement-breakpoint +-- Add embedding column to doubts table (store a fixed-length vector) +-- Using 1536 dimensions aligns with common embedding models; if you switch models, +-- update this migration and the embedding generation code accordingly. +ALTER TABLE "doubts" +ADD COLUMN IF NOT EXISTS "embedding" vector(1536); + +--> statement-breakpoint +-- Similarity index. +-- Prefer HNSW when supported by the installed pgvector version/extension. +-- If HNSW isn't available, fall back to IVFFLAT (supported in older pgvector versions). +DO $$ +BEGIN + -- If the extension supports HNSW, create the HNSW index. + -- Feature detection is best-effort: if HNSW creation fails, we fall back to IVFFLAT. + -- (Different pgvector versions expose different catalog objects, so a strict check is brittle.) + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc + WHERE proname ILIKE '%hnsw%' + ) THEN + + -- Best-effort: if HNSW support exists, create the HNSW index. + EXECUTE 'CREATE INDEX IF NOT EXISTS "doubts_embedding_cosine_idx_hnsw" ' || + 'ON "doubts" USING hnsw ("embedding" vector_cosine_ops)'; + ELSE + -- Older pgvector: create IVFFLAT index. + EXECUTE 'CREATE INDEX IF NOT EXISTS "doubts_embedding_cosine_idx_ivfflat" ' || + 'ON "doubts" USING ivfflat ("embedding" vector_cosine_ops)'; + END IF; +EXCEPTION + WHEN others THEN + -- If the HNSW detection/creation fails for any reason, fall back to IVFFLAT. + EXECUTE 'CREATE INDEX IF NOT EXISTS "doubts_embedding_cosine_idx_ivfflat" ' || + 'ON "doubts" USING ivfflat ("embedding" vector_cosine_ops)'; +END +$$; + + +--> statement-breakpoint +-- Optional: btree indexes for filtering are already present (classroomId/type/createdAt). + diff --git a/src/__tests__/api/doubts.test.ts b/src/__tests__/api/doubts.test.ts index 80176ddd..75eb0084 100644 --- a/src/__tests__/api/doubts.test.ts +++ b/src/__tests__/api/doubts.test.ts @@ -249,9 +249,6 @@ describe('Doubts API Endpoints', () => { }); it('GET should support most-replied sorting', async () => { - (db.select as jest.Mock).mockImplementationOnce(() => createChainWithData([{ count: 2 }])) - .mockImplementationOnce(() => createChainWithData([mockDoubts[0], mockDoubts[1]])); - const req = new Request('http://localhost/api/doubts?subject=Physics&sort=most-replied'); const res = await GET(req) as Response; const json = await res.json(); @@ -345,3 +342,4 @@ describe('Doubts API Endpoints', () => { expect(json.subject).toBe('Physics'); }); }); + diff --git a/src/__tests__/api/rooms-members.test.ts b/src/__tests__/api/rooms-members.test.ts index 5e0c94c5..be1f0f26 100644 --- a/src/__tests__/api/rooms-members.test.ts +++ b/src/__tests__/api/rooms-members.test.ts @@ -89,6 +89,9 @@ describe('Room Members API Endpoint', () => { const res = (await GET(new Request('http://localhost/api/rooms/members?classroomId=1')))!; const json = await res.json(); + if (res.status !== 200) { + console.error("DEBUG student res:", json); + } expect(res.status).toBe(200); expect(json).toEqual({ @@ -136,6 +139,9 @@ describe('Room Members API Endpoint', () => { const res = (await GET(new Request('http://localhost/api/rooms/members?classroomId=1')))!; const json = await res.json(); + if (res.status !== 200) { + console.error("DEBUG teacher res:", json); + } expect(res.status).toBe(200); expect(json).toEqual({ diff --git a/src/app/api/doubts/check-similarity/route.ts b/src/app/api/doubts/check-similarity/route.ts index 278ee59e..711d8c0c 100644 --- a/src/app/api/doubts/check-similarity/route.ts +++ b/src/app/api/doubts/check-similarity/route.ts @@ -3,11 +3,9 @@ import { doubtsTable, repliesTable } from "@/configs/schema"; import { and, eq, isNull, desc, inArray } from "drizzle-orm"; import { NextResponse } from "next/server"; import Groq from "groq-sdk"; -import { - buildAiProviderErrorResponse, - enforceAiAvailability, -} from "@/lib/ai/kill-switch"; +import { findSemanticDuplicates } from "@/lib/ai/embeddings"; import { buildErrorResponse } from "@/lib/error-handler"; +import { enforceAiAvailability, buildAiProviderErrorResponse } from "@/lib/ai/kill-switch"; import { getAnonymousQuotaIdentifier } from "@/lib/request-identity"; import { getSafeErrorDetails } from "@/lib/safe-error-details"; import { @@ -53,7 +51,28 @@ export async function POST(req: Request) { return NextResponse.json({ similarDoubts: [] }); } - // Fetch the last 20 doubts from the same room/community + // 1) Fast path: embedding + vector similarity search (pgvector) + // Only short-circuit when semantic search actually returns usable results. + // Empty results can mean embedding generation failed (safeGenerateEmbedding -> null) + // or no candidate passed thresholds; in those cases we must fall back to the LLM. + try { + const similarDoubts = await findSemanticDuplicates({ + content, + classroomId: classroomId ?? null, + type: "community", + similarityThreshold: 80, // 0..100 percentage contract + topK: 5, + }); + + if (similarDoubts.length > 0) { + return NextResponse.json({ similarDoubts }); + } + } catch (err) { + console.error("Embedding similarity path failed, falling back to LLM:", err); + } + + + // 2) Fallback: Fetch the last 50 doubts from the same room/community const recentDoubts = await db .select({ id: doubtsTable.id, @@ -74,6 +93,7 @@ export async function POST(req: Request) { .orderBy(desc(doubtsTable.createdAt)) .limit(50); + if (recentDoubts.length === 0) { return NextResponse.json({ similarDoubts: [] }); } diff --git a/src/app/api/doubts/route.ts b/src/app/api/doubts/route.ts index f668cbe4..d946d651 100644 --- a/src/app/api/doubts/route.ts +++ b/src/app/api/doubts/route.ts @@ -10,7 +10,8 @@ import { membershipsTable, } from "@/configs/schema"; import { categorizeDoubt } from "@/lib/ai/categorizer"; -import { and, eq, inArray, isNull, or, not, sql, SQL, desc, getTableColumns } from "drizzle-orm"; +import { safeGenerateEmbedding } from "@/lib/ai/embeddings"; +import { and, eq, inArray, isNull, or, not, sql, SQL, ilike, desc, getTableColumns } from "drizzle-orm"; import { moderateContent, handleModerationViolation } from "@/lib/moderation"; import { buildErrorResponse, errorResponse } from "@/lib/error-handler"; import { checkUserBlock } from "@/lib/auth-utils"; @@ -322,10 +323,25 @@ export async function POST(req: Request) { imageUrl, classroomId: parsedClassroomId, type: doubtType, - createdAt: parsedCreatedAt }) .returning(); + // Generate and persist embedding for semantic duplicate detection. + // Fail open: doubt creation should not block if embeddings are unavailable. + try { + const embeddingInput = `${subject}\n${content || ""}`.trim(); + const embedding = await safeGenerateEmbedding(embeddingInput); + if (embedding && Array.isArray(embedding) && embedding.length > 0) { + await db + .update(doubtsTable) + .set({ embedding: embedding as any }) + .where(eq(doubtsTable.id, newDoubt.id)); + } + } catch (err) { + console.error("Failed to generate/store doubt embedding:", err); + } + + if (parsedClassroomId) { inngest.send({ name: "doubt/created", diff --git a/src/app/api/replies/route.ts b/src/app/api/replies/route.ts index a2cfea65..b7333905 100644 --- a/src/app/api/replies/route.ts +++ b/src/app/api/replies/route.ts @@ -58,18 +58,15 @@ export async function GET(req: Request) { } if (doubt.type === 'teacher') { - if (!doubt.classroomId) { - return errorResponse("Access denied", 403); - } - const [membership] = await db - .select() - .from(membershipsTable) - .where( - and( - eq(membershipsTable.userEmail, email!), - eq(membershipsTable.classroomId, doubt.classroomId) - ) - ); + const [membership] = await db + .select() + .from(membershipsTable) + .where( + and( + eq(membershipsTable.userEmail, email as string), + eq(membershipsTable.classroomId, doubt.classroomId as number) + ) + ); const isTeacher = membership ? canTeach(membership.role) : false; const isOwner = doubt.userEmail === email; diff --git a/src/app/api/rooms/members/route.ts b/src/app/api/rooms/members/route.ts index b933e982..2d2636a8 100644 --- a/src/app/api/rooms/members/route.ts +++ b/src/app/api/rooms/members/route.ts @@ -34,7 +34,7 @@ export async function GET(req: Request) { const page = Math.max(Number(searchParams.get('page')) || 1, 1); const limit = Math.min(Math.max(Number(searchParams.get('limit')) || 20, 1), 100); const offset = (page - 1) * limit; - + const membership = await requireMembership(email, classroomId); // Total members count @@ -43,9 +43,8 @@ export async function GET(req: Request) { .from(membershipsTable) .where(eq(membershipsTable.classroomId, classroomId)); - const total = totalMembersResult[0].count; + const total = totalMembersResult[0]?.count || 0; - // Fetch paginated members of this classroom const members = await db .select({ diff --git a/src/app/ask-ai/page.tsx b/src/app/ask-ai/page.tsx index 74dba11e..3f07d52e 100644 --- a/src/app/ask-ai/page.tsx +++ b/src/app/ask-ai/page.tsx @@ -327,7 +327,8 @@ export default function AskAIPage() { Ask AI Solver -
+
+
@@ -411,7 +412,7 @@ export default function AskAIPage() { ) : ( <> - + {!imageBase64 ? (
)} +
diff --git a/src/configs/db.tsx b/src/configs/db.tsx index 20cf93ac..8b59a134 100644 --- a/src/configs/db.tsx +++ b/src/configs/db.tsx @@ -5,3 +5,4 @@ export const db = drizzle(getDatabaseUrl()); /** Re-export the transaction helper so callers import from one place. */ export { db as default }; + diff --git a/src/configs/schema.ts b/src/configs/schema.ts index 31f2c2cc..1c7d72d3 100644 --- a/src/configs/schema.ts +++ b/src/configs/schema.ts @@ -1,5 +1,5 @@ // configs/schema.ts -import { integer, pgTable, varchar, text, timestamp, boolean, index, uniqueIndex, foreignKey, unique } from "drizzle-orm/pg-core"; +import { integer, pgTable, varchar, text, timestamp, boolean, index, uniqueIndex, foreignKey, unique, vector } from "drizzle-orm/pg-core"; export const usersTable = pgTable("users", { id: integer().primaryKey().generatedAlwaysAsIdentity(), @@ -194,6 +194,10 @@ export const doubtsTable = pgTable("doubts", { isPinned: boolean().default(false), deletedAt: timestamp(), createdAt: timestamp().defaultNow().notNull(), + + // Semantic duplicate detection + // NOTE: stored as pgvector embedding(1536) + embedding: vector({ dimensions: 1536 }), }, (table) => { return { classroomIdIndex: index("doubt_classroomId_idx").on(table.classroomId), diff --git a/src/lib/ai/embeddings.ts b/src/lib/ai/embeddings.ts new file mode 100644 index 00000000..d41a7b43 --- /dev/null +++ b/src/lib/ai/embeddings.ts @@ -0,0 +1,173 @@ +import Groq from "groq-sdk"; +import { db } from "@/configs/db"; +import { doubtsTable, repliesTable } from "@/configs/schema"; +import { and, eq, isNull, desc, inArray, SQL, sql } from "drizzle-orm"; + +const groq = new Groq({ + apiKey: process.env.GROQ_API_KEY || "dummy_key", +}); + +const EMBEDDING_DIMENSIONS = 1536; +const DEFAULT_SIMILARITY_THRESHOLD = 0.8; // cosine similarity +const DEFAULT_TOP_K = 5; + +function getEmbeddingInput(content: string, subject?: string | null) { + const c = content?.trim(); + const s = subject?.trim(); + if (s) return `${s}\n${c}`; + return c || ""; +} + +async function generateGroqEmbedding(textInput: string): Promise { + const trimmed = textInput.trim(); + if (!trimmed) throw new Error("Empty embedding text"); + + // Groq embeddings API returns vectors under `embedding`. + // Docs: https://console.groq.com/docs/embeddings + const res = await groq.embeddings.create({ + model: "nomic-embed-text", + input: trimmed, + encoding_format: "float", + }); + + const vector = (res.data?.[0]?.embedding || []) as number[]; + + if (!Array.isArray(vector) || vector.length === 0) { + throw new Error("Groq returned empty embedding"); + } + + if (vector.length !== EMBEDDING_DIMENSIONS) { + throw new Error( + `Groq returned embedding dimension ${vector.length}, expected ${EMBEDDING_DIMENSIONS}`, + ); + } + + return vector; +} + +/** + * Returns embedding vector for a text, or null if generation fails. + */ +export async function safeGenerateEmbedding( + textInput: string, +): Promise { + try { + const vector = await generateGroqEmbedding(textInput); + return vector; + } catch (err) { + console.error("Embedding generation failed:", err); + return null; + } +} + +export interface SemanticDuplicateCandidate { + id: number; + subject: string; + content: string | null; + isSolved: string | null; + similarity: number; + solvedAnswer?: string | null; +} + +/** + * Vector search across full doubt history for the same scope. + * + * Cosine similarity is computed using pgvector operator: + * embedding <=> query_embedding + * returns (1 - cosine_similarity). Therefore cosine = 1 - distance. + */ +export async function findSemanticDuplicates(params: { + content: string; + classroomId?: number | null; + subject?: string | null; + similarityThreshold?: number; + topK?: number; + // Only search community doubts by default to match existing behavior + type?: string; +}): Promise { + const { + content, + classroomId, + subject, + similarityThreshold = DEFAULT_SIMILARITY_THRESHOLD, + topK = DEFAULT_TOP_K, + type = "community", + } = params; + + const embeddingInput = getEmbeddingInput(content, subject); + const queryEmbedding = await safeGenerateEmbedding(embeddingInput); + if (!queryEmbedding) return []; + + // pgvector expects an array of numbers cast to vector. + // We pass as a SQL parameter via `sql`. + const queryVec = sql`${queryEmbedding}::vector`; + + const whereClause = and( + classroomId + ? eq(doubtsTable.classroomId, classroomId) + : isNull(doubtsTable.classroomId), + eq(doubtsTable.type, type), + // exclude null embeddings + sql`${doubtsTable.embedding} IS NOT NULL`, + ); + + // pgvector cosine distance: (embedding <=> query) returns distance in [0..2] depending on normalization. + // Existing code treated (1 - distance) as similarity, which is incorrect for a 0..100 % contract. + // Convert cosine similarity (approx in [0..1]) to a 0..100 percentage. + // Note: We clamp to [0,100] defensively. + const similarityExpr = sql`( + greatest(0, least(100, (1 - (${doubtsTable.embedding} <=> ${queryVec})) * 100)) + )`; + + + + + const rows = await db + + .select({ + id: doubtsTable.id, + subject: doubtsTable.subject, + content: doubtsTable.content, + isSolved: doubtsTable.isSolved, + similarity: similarityExpr, + solvedReplyId: doubtsTable.solvedReplyId, + }) + .from(doubtsTable) + .where(whereClause) + .orderBy(sql`${doubtsTable.embedding} <=> ${queryVec} ASC`) + .limit(topK); + + + const filtered = rows + .filter((r) => typeof r.similarity === "number" && r.similarity >= similarityThreshold) + .sort((a, b) => (b.similarity ?? 0) - (a.similarity ?? 0)); + + const solvedReplyIds = filtered + .filter((d) => d.isSolved === "solved" && d.solvedReplyId) + .map((d) => d.solvedReplyId as number); + + const solvedReplies = + solvedReplyIds.length > 0 + ? await db + .select({ id: repliesTable.id, content: repliesTable.content }) + .from(repliesTable) + .where(inArray(repliesTable.id, solvedReplyIds)) + : []; + + const replyMap = new Map( + solvedReplies.map((r) => [r.id, r.content]), + ); + + return filtered.map((d) => ({ + id: d.id, + subject: d.subject, + content: d.content, + isSolved: d.isSolved, + similarity: d.similarity, + solvedAnswer: + d.isSolved === "solved" && d.solvedReplyId + ? replyMap.get(d.solvedReplyId) ?? null + : null, + })); +} +