feat: moderation v2 trust verification pipeline

convex/lib/moderation.ts

@@ -1,49 +1,49 @@
-import type { Doc } from '../_generated/dataModel'
+import type { Doc } from "../_generated/dataModel";
 
-const FLAG_RULES: Array<{ flag: string; pattern: RegExp }> = [
-  // Known-bad / known-suspicious identifiers.
-  // NOTE: keep these narrowly scoped; use staff review to confirm removals.
-  {
-    flag: 'blocked.malware',
-    pattern: /(keepcold131\/ClawdAuthenticatorTool|ClawdAuthenticatorTool)/i,
-  },
-
-  { flag: 'suspicious.keyword', pattern: /(malware|stealer|phish|phishing|keylogger)/i },
-  { flag: 'suspicious.secrets', pattern: /(api[-_ ]?key|token|password|private key|secret)/i },
-  { flag: 'suspicious.crypto', pattern: /(wallet|seed phrase|mnemonic|crypto)/i },
-  { flag: 'suspicious.webhook', pattern: /(discord\.gg|webhook|hooks\.slack)/i },
-  { flag: 'suspicious.script', pattern: /(curl[^\n]+\|\s*(sh|bash))/i },
-  { flag: 'suspicious.url_shortener', pattern: /(bit\.ly|tinyurl\.com|t\.co|goo\.gl|is\.gd)/i },
-]
+const KNOWN_BLOCKED_SIGNATURE = /(keepcold131\/ClawdAuthenticatorTool|ClawdAuthenticatorTool)/i;
+const SUSPICIOUS_INSTALL_URL = /https?:\/\/(bit\.ly|tinyurl\.com|t\.co|goo\.gl|is\.gd)\//i;
+const SUSPICIOUS_RAW_IP_URL = /https?:\/\/\d{1,3}(?:\.\d{1,3}){3}/i;
+const SUSPICIOUS_SCRIPT_PIPE = /curl[^\n]+\|\s*(sh|bash)/i;
 
 export function deriveModerationFlags({
   skill,
   parsed,
   files,
 }: {
-  skill: Pick<Doc<'skills'>, 'slug' | 'displayName' | 'summary'>
-  parsed: Doc<'skillVersions'>['parsed']
-  files: Doc<'skillVersions'>['files']
+  skill: Pick<Doc<"skills">, "slug" | "displayName" | "summary">;
+  parsed: Doc<"skillVersions">["parsed"];
+  files: Doc<"skillVersions">["files"];
 }) {
   const text = [
     skill.slug,
     skill.displayName,
-    skill.summary ?? '',
+    skill.summary ?? "",
     JSON.stringify(parsed?.frontmatter ?? {}),
     JSON.stringify(parsed?.metadata ?? {}),
     JSON.stringify((parsed as { moltbot?: unknown } | undefined)?.moltbot ?? {}),
     ...files.map((file) => file.path),
   ]
     .filter(Boolean)
-    .join('\n')
+    .join("\n");
+
+  const flags = new Set<string>();
+  if (KNOWN_BLOCKED_SIGNATURE.test(text)) {
+    flags.add("blocked.malware");
+  }
 
-  const flags = new Set<string>()
+  // Context-aware suspicious checks only. Avoid broad keyword-only flags to reduce false positives.
+  if (
+    SUSPICIOUS_INSTALL_URL.test(text) ||
+    SUSPICIOUS_RAW_IP_URL.test(text) ||
+    SUSPICIOUS_SCRIPT_PIPE.test(text)
+  ) {
+    flags.add("flagged.suspicious");
+  }
 
-  for (const rule of FLAG_RULES) {
-    if (rule.pattern.test(text)) {
-      flags.add(rule.flag)
-    }
+  const always = (parsed?.frontmatter as Record<string, unknown> | undefined)?.always;
+  if (always === true || always === "true") {
+    flags.add("flagged.suspicious");
   }
 
-  return Array.from(flags)
+  return Array.from(flags);
 }

convex/lib/moderationEngine.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { buildModerationSnapshot, runStaticModerationScan } from "./moderationEngine";
+
+describe("moderationEngine", () => {
+  it("does not flag benign token/password docs text alone", () => {
+    const result = runStaticModerationScan({
+      slug: "demo",
+      displayName: "Demo",
+      summary: "A normal integration skill",
+      frontmatter: {},
+      metadata: {},
+      files: [{ path: "SKILL.md", size: 64 }],
+      fileContents: [
+        {
+          path: "SKILL.md",
+          content:
+            "This skill requires API token and password from the official provider settings.",
+        },
+      ],
+    });
+    expect(result.reasonCodes).toEqual([]);
+    expect(result.status).toBe("clean");
+  });
+
+  it("flags dynamic eval usage as suspicious", () => {
+    const result = runStaticModerationScan({
+      slug: "demo",
+      displayName: "Demo",
+      summary: "A normal integration skill",
+      frontmatter: {},
+      metadata: {},
+      files: [{ path: "index.ts", size: 64 }],
+      fileContents: [{ path: "index.ts", content: "const value = eval(code)" }],
+    });
+    expect(result.reasonCodes).toContain("suspicious.dynamic_code_execution");
+    expect(result.status).toBe("suspicious");
+  });
+
+  it("upgrades merged verdict to malicious when VT is malicious", () => {
+    const snapshot = buildModerationSnapshot({
+      staticScan: {
+        status: "suspicious",
+        reasonCodes: ["suspicious.dynamic_code_execution"],
+        findings: [],
+        summary: "",
+        engineVersion: "v2.0.0",
+        checkedAt: Date.now(),
+      },
+      vtStatus: "malicious",
+    });
+    expect(snapshot.verdict).toBe("malicious");
+    expect(snapshot.reasonCodes).toContain("malicious.vt_malicious");
+  });
+});