Implemented semantic duplicate detection for doubts using embeddings

.github/workflows/pr-code-quality-commenter.yml

@@ -0,0 +1,70 @@
+name: PR Code Quality Commenter
+
+on:
+  workflow_run:
+    workflows: ["PR Code Quality Checks"]
+    types:
+      - completed
+
+permissions:
+  pull-requests: write
+  issues: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          path: pr-comments
+        continue-on-error: true
+
+      - name: Post Comments
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            // Find the PR associated with this run
+            const pulls = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              commit_sha: context.payload.workflow_run.head_sha
+            });
+            const pr = pulls.data[0];
+            if (!pr) {
+              console.log("No pull request found for commit: " + context.payload.workflow_run.head_sha);
+              return;
+            }
+            const prNumber = pr.number;
+            console.log(`Associated PR: #${prNumber}`);
+
+            const postComment = async (dirName, fileName) => {
+              const dirPath = path.join(process.env.GITHUB_WORKSPACE, 'pr-comments', dirName);
+              if (!fs.existsSync(dirPath)) {
+                console.log(`Directory ${dirName} does not exist. Skipping.`);
+                return;
+              }
+
+              const commentFile = path.join(dirPath, fileName);
+              if (fs.existsSync(commentFile)) {
+                const body = fs.readFileSync(commentFile, 'utf8').trim();
+                if (body) {
+                  console.log(`Posting comment from ${dirName} to PR #${prNumber}...`);
+                  await github.rest.issues.createComment({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    issue_number: prNumber,
+                    body: body
+                  });
+                }
+              }
+            };
+
+            await postComment('security-warning', 'security-warning.txt');
+            await postComment('quality-gate-warning', 'quality-gate-warning.txt');

.github/workflows/pr-code-quality.yml

@@ -0,0 +1,261 @@
+name: PR Code Quality Checks
+
+on:
+  pull_request:
+    branches: [main, master]
+    types: [opened, synchronize, reopened]
+
+permissions: {}
+
+jobs:
+  # ─────────────────────────────────────────
+  # 1. TypeScript type checking
+  # ─────────────────────────────────────────
+  typecheck:
+    name: TypeScript Check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20.x"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run TypeScript compiler check
+        run: npx tsc --noEmit
+        continue-on-error: true
+
+  # ─────────────────────────────────────────
+  # 2. ESLint - catch code issues
+  # ─────────────────────────────────────────
+  lint:
+    name: ESLint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20.x"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run ESLint
+        run: npx eslint . --ext .ts,.tsx --max-warnings=20
+        continue-on-error: true
+
+  # ─────────────────────────────────────────
+  # 3. Build check — does it compile?
+  # ─────────────────────────────────────────
+  build:
+    name: Build Check
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    env:
+      # Dummy env vars so Next.js build doesn't fail on missing secrets
+      NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${{ secrets.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY || 'pk_test_placeholder' }}
+      CLERK_SECRET_KEY: ${{ secrets.CLERK_SECRET_KEY || 'sk_test_placeholder' }}
+      DATABASE_URL: ${{ secrets.DATABASE_URL || 'postgresql://placeholder' }}
+      NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'https://placeholder.supabase.co' }}
+      NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'placeholder' }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20.x"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Next.js app
+        run: npm run build
+        continue-on-error: true
+
+  # ─────────────────────────────────────────
+  # 4. Security scan — detect common issues
+  # ─────────────────────────────────────────
+  security-scan:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { data: files } = await github.rest.pulls.listFiles({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+            return files.map(f => f.filename);
+
+      - name: Scan for hardcoded secrets patterns
+        id: secret-scan
+        run: |
+          echo "Scanning for potential secrets in changed files..."
+          ISSUES=""
+
+          # Check for common secret patterns
+          if git diff origin/main...HEAD -- '*.ts' '*.tsx' '*.js' | grep -E "(password|secret|api_key|apikey|token)\s*=\s*['\"][^'\"]{8,}" --include="*.ts" --include="*.tsx" -i; then
+            ISSUES="$ISSUES\n- Possible hardcoded secret detected"
+          fi
+
+          # Check for NEXT_PUBLIC_ on sensitive vars
+          if git diff origin/main...HEAD | grep -E "NEXT_PUBLIC_(DATABASE_URL|SECRET|PASSWORD|PRIVATE)" -i; then
+            ISSUES="$ISSUES\n- Sensitive variable exposed via NEXT_PUBLIC_ prefix"
+          fi
+
+          # Check for console.log with sensitive data patterns
+          if git diff origin/main...HEAD -- '*.ts' '*.tsx' | grep -E "console\.(log|error)\(.*?(password|token|secret|key)" -i; then
+            ISSUES="$ISSUES\n- Possible sensitive data in console.log"
+          fi
+
+          if [ -n "$ISSUES" ]; then
+            echo "security_issues=true" >> $GITHUB_OUTPUT
+            echo "issues=$ISSUES" >> $GITHUB_OUTPUT
+          else
+            echo "security_issues=false" >> $GITHUB_OUTPUT
+          fi
+        continue-on-error: true
+
+      - name: Save security findings comment
+        if: steps.secret-scan.outputs.security_issues == 'true'
+        uses: actions/github-script@v6
+        env:
+          SECURITY_ISSUES: ${{ steps.secret-scan.outputs.issues }}
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+            const dir = './security-warning';
+            if (!fs.existsSync(dir)){
+              fs.mkdirSync(dir, { recursive: true });
+            }
+            const body = `## Security Scan Warning\n\nPotential security issues detected in this PR:\n${process.env.SECURITY_ISSUES}\n\nPlease review before merging. @knoxiboy`;
+            fs.writeFileSync(path.join(dir, 'security-warning.txt'), body);
+
+
+      - name: Upload security findings artifact
+        if: steps.secret-scan.outputs.security_issues == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: security-warning
+          path: security-warning/
+          if-no-files-found: ignore
+
+  # ─────────────────────────────────────────
+  # 5. PR quality gate — basic checks
+  # ─────────────────────────────────────────
+  pr-quality-gate:
+    name: PR Quality Gate
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check PR quality
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const pr = context.payload.pull_request;
+            const title = pr.title || '';
+            const body = pr.body || '';
+            const issues = [];
+
+            // 1. Title format check
+            const validPrefixes = ['feat', 'fix', 'docs', 'style', 'refactor', 'test', 'chore', 'perf', 'ci', 'build', 'revert'];
+            const hasValidPrefix = validPrefixes.some(p => title.toLowerCase().startsWith(p));
+            if (!hasValidPrefix) {
+              issues.push(`**Title format**: Title should start with a conventional commit prefix (e.g. \`feat:\`, \`fix:\`, \`docs:\`). Current: \`${title}\``);
+            }
+
+            // 2. PR description check
+            if (body.trim().length < 50) {
+              issues.push('**Description**: PR description is too short. Please describe what changes were made and why.');
+            }
+
+            // 3. Check for linked issue
+            const hasLinkedIssue = /closes?\s+#\d+|fixes?\s+#\d+|resolves?\s+#\d+/i.test(body);
+            if (!hasLinkedIssue) {
+              issues.push('**Linked Issue**: No linked issue found. Please add `Closes #<issue-number>` to your PR description.');
+            }
+
+            // 4. Check for test mention
+            const hasTestMention = /test|spec|jest|vitest|playwright/i.test(body);
+            if (!hasTestMention) {
+              issues.push('**Testing**: No mention of tests in the PR description. Please describe how you tested your changes.');
+            }
+
+            // Post results
+            if (issues.length > 0) {
+              const comment = `## PR Quality Check\n\nThe following items need attention:\n\n${issues.map(i => `- ${i}`).join('\n')}\n\n> These are suggestions to improve PR quality. The PR can still be merged after review.`;
+              const fs = require('fs');
+              const path = require('path');
+              const dir = './quality-gate-warning';
+              if (!fs.existsSync(dir)){
+                fs.mkdirSync(dir, { recursive: true });
+              }
+              fs.writeFileSync(path.join(dir, 'quality-gate-warning.txt'), comment);
+              console.log('Saved quality gate feedback to file');
+            } else {
+              console.log('PR passed all quality checks');
+            }
+
+      - name: Upload quality gate warning artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: quality-gate-warning
+          path: quality-gate-warning/
+          if-no-files-found: ignore
+
+  # ─────────────────────────────────────────
+  # 6. Dependency audit
+  # ─────────────────────────────────────────
+  dependency-audit:
+    name: Dependency Audit
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20.x"
+          cache: "npm"
+
+      - name: Check for new dependencies added
+        id: dep-check
+        run: |
+          git diff origin/main...HEAD -- package.json | grep "^+" | grep -v "^+++" | grep -E '"[^"]+"\s*:\s*"' || echo "No new dependencies"
+
+      - name: Run npm audit on high severity
+        run: npm audit --audit-level=high
+        continue-on-error: true

drizzle/0009_add_doubt_embeddings_pgvector.sql

@@ -0,0 +1,45 @@
+-- Enable pgvector for embedding storage and similarity search
+CREATE EXTENSION IF NOT EXISTS vector;
+
+--> statement-breakpoint
+-- Add embedding column to doubts table (store a fixed-length vector)
+-- Using 1536 dimensions aligns with common embedding models; if you switch models,
+-- update this migration and the embedding generation code accordingly.
+ALTER TABLE "doubts"
+ADD COLUMN IF NOT EXISTS "embedding" vector(1536);
+
+--> statement-breakpoint
+-- Similarity index.
+-- Prefer HNSW when supported by the installed pgvector version/extension.
+-- If HNSW isn't available, fall back to IVFFLAT (supported in older pgvector versions).
+DO $$
+BEGIN
+  -- If the extension supports HNSW, create the HNSW index.
+  -- Feature detection is best-effort: if HNSW creation fails, we fall back to IVFFLAT.
+  -- (Different pgvector versions expose different catalog objects, so a strict check is brittle.)
+  IF EXISTS (
+    SELECT 1
+    FROM pg_catalog.pg_proc
+    WHERE proname ILIKE '%hnsw%'
+  ) THEN
+
+    -- Best-effort: if HNSW support exists, create the HNSW index.
+    EXECUTE 'CREATE INDEX IF NOT EXISTS "doubts_embedding_cosine_idx_hnsw" ' ||
+            'ON "doubts" USING hnsw ("embedding" vector_cosine_ops)';
+  ELSE
+    -- Older pgvector: create IVFFLAT index.
+    EXECUTE 'CREATE INDEX IF NOT EXISTS "doubts_embedding_cosine_idx_ivfflat" ' ||
+            'ON "doubts" USING ivfflat ("embedding" vector_cosine_ops)';
+  END IF;
+EXCEPTION
+  WHEN others THEN
+    -- If the HNSW detection/creation fails for any reason, fall back to IVFFLAT.
+    EXECUTE 'CREATE INDEX IF NOT EXISTS "doubts_embedding_cosine_idx_ivfflat" ' ||
+            'ON "doubts" USING ivfflat ("embedding" vector_cosine_ops)';
+END
+$$;
+
+
+--> statement-breakpoint
+-- Optional: btree indexes for filtering are already present (classroomId/type/createdAt).
+

src/__tests__/api/doubts.test.ts

@@ -249,9 +249,6 @@ describe('Doubts API Endpoints', () => {
 });
 
     it('GET should support most-replied sorting', async () => {
-        (db.select as jest.Mock).mockImplementationOnce(() => createChainWithData([{ count: 2 }]))
-                                .mockImplementationOnce(() => createChainWithData([mockDoubts[0], mockDoubts[1]]));
-
         const req = new Request('http://localhost/api/doubts?subject=Physics&sort=most-replied');
         const res = await GET(req) as Response;
         const json = await res.json();
@@ -345,3 +342,4 @@ describe('Doubts API Endpoints', () => {
         expect(json.subject).toBe('Physics');
     });
 });
+