scraper.ts

import * as dotenv from "dotenv";
import fs from "fs";
import readline from "readline";
import { promisify } from "util";
import { fetchCommentReactions, fetchWithRateLimit } from "./fetch-utils";
import { reviewBug } from "./review-utils";
import { assessBugDifficulty } from "./review-utils-difficulty";

// Load environment variables
dotenv.config();

// Define the Bug type
interface Bug {
  repo: string;
  prNumber: number;
  prTitle: string;
  prAuthor: string;
  prLink: string;
  commentAuthor: string;
  commentBody: string;
  commentBodyIncludesCodeSnippet: boolean;
  commentLink: string;
  filePath: string;
  diffHunk: string;
  line: number;
  reactionCount: number;
  reactions: any[];
  timestamp: string;
  isReviewComment: boolean;
  pullRequestReviewId: string;
  LLMReviewed?: boolean;
  LLMReviewPassed?: boolean;
  difficultyLevel?: string;
}

const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
if (!GITHUB_TOKEN)
  throw new Error("GITHUB_TOKEN is not set in the environment or .env file");

// Configuration
const PER_REPO_MAX_BUGS = 3000; // Stop after finding this many bugs per repository
const OUTPUT_FILE = "bugs.json";
const SCANNED_FILE = "scanned_prs.txt";

const REPOS = [
  // some js repos
  // { owner: "axios", repo: "axios" },
  // electron
  // { owner: "electron", repo: "electron" },
  // vscode
  // scraped until #175402 in microsoft/vscode...
  // { owner: "microsoft", repo: "vscode" },

  // react-native
  // { owner: "facebook", repo: "react-native" },
  // node
  // { owner: "nodejs", repo: "node" },

  // some python repos
  // { owner: "pandas-dev", repo: "pandas" },
  // { owner: "numpy", repo: "numpy" },
  // { owner: "scikit-learn", repo: "scikit-learn" },
  // { owner: "tensorflow", repo: "tensorflow" },
  // { owner: "pytorch", repo: "pytorch" },

  // some go repos
  // { owner: "kubernetes", repo: "kubernetes" },
  // { owner: "golang", repo: "go" },
  // { owner: "etcd-io", repo: "etcd" },
  // { owner: "grpc", repo: "grpc-go" },
  // { owner: "grpc", repo: "grpc-java" },

  // some popular rust repos
  // { owner: "rust-lang", repo: "rust" },
  // { owner: "tokio-rs", repo: "tokio" },
  // { owner: "serde-rs", repo: "serde" },
  // { owner: "actix", repo: "actix-web" },

  // some more popular c/c++ repos
  // { owner: "ggerganov", repo: "llama.cpp" },
  // { owner: "opencv", repo: "opencv" },
  // { owner: "godotengine", repo: "godot" },

  // some more ts repos
  // { owner: "vercel", repo: "next.js" },
  // { owner: "vitejs", repo: "vite" },
  // java
  // { owner: "apache", repo: "kafka" },

  // lua
  // { owner: "LuaDist", repo: "lua" },

  // elixir
  // { owner: "elixir-lang", repo: "elixir" },
  // { owner: "phoenixframework", repo: "phoenix" },
  // { owner: "anoma", repo: "anoma" },
  // { owner: "electric-sql", repo: "electric" },

  // { owner: "sveltejs", repo: "svelte" },
  // { owner: "vuejs", repo: "core" },
  // { owner: "nestjs", repo: "nest" },
  // { owner: "prisma", repo: "prisma" },
  // { owner: "remix-run", repo: "remix" },

  // Popular web frameworks
  // { owner: "rails", repo: "rails" },
  // { owner: "laravel", repo: "laravel" },
  // { owner: "django", repo: "django" },
  // { owner: "spring-projects", repo: "spring-boot" },

  // Developer tools
  // { owner: "docker", repo: "compose" },

  // prob not good bc very sophisticated devs here

// { owner: "angular", repo: "angular" },
// { owner: "DefinitelyTyped", repo: "DefinitelyTyped" },
// { owner: "palantir", repo: "blueprint" },

// lets try these
// { owner: "storybookjs", repo: "storybook" },
// { owner: "grafana", repo: "grafana" },
// { owner: "langgenius", repo: "dify" },
// { owner: "continuedev", repo: "continue" },
// { owner: "infiniflow", repo: "ragflow" },
// { owner: "Budibase", repo: "budibase" },
// { owner: "siyuan-note", repo: "siyuan" },
// { owner: "epicweb-dev", repo: "epic-stack" },
// { owner: "midday-ai", repo: "midday" },
// { owner: "TheAlgorithms", repo: "Typescript" },
// { owner: "triggerdotdev", repo: "trigger.dev" },
// { owner: "formbricks", repo: "formbricks" },
// { owner: "excalidraw", repo: "excalidraw" },
// { owner: "pmndrs", repo: "react-three-fiber" },
// { owner: "chakra-ui", repo: "chakra-ui" },
// { owner: "apollographql", repo: "apollo-client" },
// { owner: "wagmi-dev", repo: "wagmi" },
// { owner: "drizzle-team", repo: "drizzle-orm" },
// { owner: "mantinedev", repo: "mantine" },
// { owner: "xyflow", repo: "react-flow" },
// { owner: "jaredpalmer", repo: "formik" },
// { owner: "emotion-js", repo: "emotion" },
// { owner: "pmmmwh", repo: "react-refresh-webpack-plugin" },
// { owner: "react-hook-form", repo: "react-hook-form" },
// { owner: "testing-library", repo: "react-testing-library" },
// { owner: "shoelace-style", repo: "shoelace" },
// { owner: "radix-ui", repo: "primitives" },

  // random languages - maybe later
  // { owner: "hashicorp", repo: "terraform" },
  // { owner: "git", repo: "git" },
  // { owner: "neovim", repo: "neovim" },

  // Databases
  // { owner: "mongodb", repo: "mongo" },
  // { owner: "redis", repo: "redis" },

  // more js related db projects
  // { owner: "sequelize", repo: "sequelize" },
  // { owner: "knex", repo: "knex" },

  // more python related db projects
  // { owner: "sqlalchemy", repo: "sqlalchemy" },
  // { owner: "tortoise", repo: "tortoise-orm" },
  // { owner: "sqlmodel", repo: "sqlmodel" },

  // Mobile development js or ts 
  // { owner: "flutter", repo: "flutter" },
  // { owner: "expo", repo: "expo" },
  // { owner: "react-navigation", repo: "react-navigation" },
  
  // javascript (ones that are not listed above already)
  // { owner: "expressjs", repo: "express" },
  // { owner: "jestjs", repo: "jest" },
  // { owner: "mochajs", repo: "mocha" },
  // { owner: "chaijs", repo: "chai" },

  // javascript related but not in the list above
  // { owner: "webpack", repo: "webpack" },
  // { owner: "rollup", repo: "rollup" },
  // { owner: "parcel-bundler", repo: "parcel" },

  // Popular JavaScript UI libraries and frameworks
  // { owner: "twbs", repo: "bootstrap" },
  // { owner: "tailwindlabs", repo: "tailwindcss" },
  // { owner: "quilljs", repo: "quill" },
  // { owner: "lukeed", repo: "clsx" },
  // { owner: "kamranahmedse", repo: "driver.js" },
  // { owner: "saadeghi", repo: "daisyui" },
  
  // JavaScript utilities and tools
  // { owner: "plausible", repo: "analytics" },
  // { owner: "janhq", repo: "jan" },
  // { owner: "bitcoinjs", repo: "bitcoinjs-lib" },
  // { owner: "pmndrs", repo: "zustand" },
  // { owner: "laurent22", repo: "joplin" },
  
  // JavaScript backend and full-stack
  // { owner: "withastro", repo: "astro" },
  // { owner: "nocodb", repo: "nocodb" },
  // { owner: "appwrite", repo: "appwrite" },
  // { owner: "directus", repo: "directus" },
  { owner: "PostHog", repo: "posthog" },
  { owner: "GladysAssistant", repo: "Gladys" },

  // System programming
  { owner: "torvalds", repo: "linux" },
  { owner: "apple", repo: "swift" },
  { owner: "JetBrains", repo: "kotlin" },

  // swift
  { owner: "apple", repo: "swift-algorithms" },

];

const HEADERS = {
  Authorization: `Bearer ${GITHUB_TOKEN}`,
  Accept: "application/vnd.github+json",
};

// Reactions to consider
const ALLOWED_REACTIONS = ["hooray", "+1", "rocket"];

// File utilities
const readFileAsync = promisify(fs.readFile);
const writeFileAsync = promisify(fs.writeFile);
const appendFileAsync = promisify(fs.appendFile);

// Load scanned PRs efficiently
const loadScannedPRs = async (): Promise<Set<string>> => {
  const scannedPRs = new Set<string>();

  try {
    // Create read interface
    const fileStream = fs.createReadStream(SCANNED_FILE);
    const rl = readline.createInterface({
      input: fileStream,
      crlfDelay: Infinity,
    });

    // Read line by line
    for await (const line of rl) {
      if (line.trim() && !line.includes("[") && !line.includes("]")) {
        scannedPRs.add(line.replace(/[",]/g, "").trim());
      }
    }

    return scannedPRs;
  } catch {
    return new Set();
  }
};

// Save single PR to scanned list
const saveSinglePR = async (prKey: string): Promise<void> => {
  await appendFileAsync(SCANNED_FILE, prKey + "\n");
};

// Save scanned PRs efficiently
const saveScannedPRs = async (scannedPRs: Set<string>): Promise<void> => {
  // Only use this for initial save or final backup
  await writeFileAsync(SCANNED_FILE, Array.from(scannedPRs).join("\n") + "\n");
};

// Append to output file
const appendToOutputFile = async (data: Bug[]): Promise<void> => {
  try {
    let existingData: Bug[] = [];
    try {
      existingData = JSON.parse(await readFileAsync(OUTPUT_FILE, "utf8"));
    } catch {
      // File doesn't exist or is empty, start with empty array
    }

    // Review each bug as it's found
    const passedBugs: Bug[] = [];
    for (const bug of data) {
      console.log(`Reviewing bug in PR #${bug.prNumber} from ${bug.repo}...`);
      const reviewResult = await reviewBug(bug);
      if (reviewResult !== null) {
        bug.LLMReviewed = true;
        bug.LLMReviewPassed = reviewResult;
        console.log(
          `Review result for PR #${bug.prNumber}: ${
            reviewResult ? "PASS" : "FAIL"
          }`
        );

        // If the review passed, assess difficulty and add to passedBugs
        if (reviewResult) {
          console.log(`Assessing difficulty for PR #${bug.prNumber}...`);
          const difficultyLevel = await assessBugDifficulty(bug);
          if (difficultyLevel !== null) {
            bug.difficultyLevel = difficultyLevel;
            console.log(`Difficulty assessment result for PR #${bug.prNumber}: ${difficultyLevel}`);
          } else {
            console.log(`Difficulty assessment failed for PR #${bug.prNumber} (error occurred)`);
          }
          
          // Only add bugs that passed the review
          passedBugs.push(bug);
          console.log(`Bug in PR #${bug.prNumber} PASSED review and will be saved.`);
        } else {
          // console.log(`Bug in PR #${bug.prNumber} FAILED review and will NOT be saved.`);
        }
      } else {
        console.log(`Review failed for PR #${bug.prNumber} (error occurred)`);
      }
    }

    // Only save bugs that passed the review
    if (passedBugs.length > 0) {
      const newData = [...existingData, ...passedBugs];
      await writeFileAsync(OUTPUT_FILE, JSON.stringify(newData, null, 2));
      console.log(`Saved ${passedBugs.length} passing bugs to ${OUTPUT_FILE}`);
    } else {
      console.log(`No passing bugs to save.`);
    }
  } catch (error) {
    console.error(`Error appending to output file: ${error}`);
    // Only write passing bugs even if there was an error
    const passedBugs = data.filter(bug => bug.LLMReviewPassed === true);
    if (passedBugs.length > 0) {
      await writeFileAsync(OUTPUT_FILE, JSON.stringify(passedBugs, null, 2));
    }
  }
};

// Find the oldest PR number we've scanned for a repo
const findOldestScannedPR = (
  scannedPRs: Set<string>,
  owner: string,
  repo: string
): number => {
  let oldestPRNumber = Infinity;
  for (const key of scannedPRs) {
    if (key.startsWith(`${owner}/${repo}#`)) {
      const prNumber = parseInt(key.split("#")[1]);
      if (!isNaN(prNumber) && prNumber < oldestPRNumber) {
        oldestPRNumber = prNumber;
      }
    }
  }
  return oldestPRNumber === Infinity ? 0 : oldestPRNumber;
};

// Fetch pull requests
const fetchPullRequests = (
  owner: string,
  repo: string,
  page = 1,
  since?: string
): Promise<any[]> => {
  const url = `https://api.github.com/repos/${owner}/${repo}/pulls`;
  const params: {
    state: string;
    per_page: number;
    page: number;
    sort: string;
    direction: string;
    since?: string;
  } = {
    state: "closed",
    per_page: 100,
    page,
    sort: "created",
    direction: "desc",
  };
  if (since) {
    params.since = since;
  }
  return fetchWithRateLimit(url, HEADERS, params);
};

// Fetch review comments
const fetchReviewComments = (
  owner: string,
  repo: string,
  prNumber: number
): Promise<any[]> => {
  const url = `https://api.github.com/repos/${owner}/${repo}/pulls/${prNumber}/comments`;
  return fetchWithRateLimit(url, HEADERS);
};

// Fetch PR events to check for force pushes
const fetchPREvents = async (
  owner: string,
  repo: string,
  prNumber: number
): Promise<any[]> => {
  const url = `https://api.github.com/repos/${owner}/${repo}/issues/${prNumber}/events`;
  return fetchWithRateLimit(url, HEADERS);
};

// Fetch PR commits
const fetchPRCommits = async (
  owner: string,
  repo: string,
  prNumber: number
): Promise<any[]> => {
  const url = `https://api.github.com/repos/${owner}/${repo}/pulls/${prNumber}/commits`;
  return fetchWithRateLimit(url, HEADERS);
};

// Check if the comment is bug-related
const noLinksOrImages = (comment: any): boolean => {
  if (!comment.body) return false;
  const bodyLower = comment.body.toLowerCase();

  // Always skip comments containing images, issue/PR references, or links
  if (
    bodyLower.includes("![") ||
    bodyLower.includes("<img") ||
    /#\d+/.test(bodyLower) || // Matches any #1234 style reference
    /\[.+?\]\(.+?\)/.test(bodyLower) || // Matches markdown links [text](url)
    /https?:\/\/\S+/.test(bodyLower) // Matches URLs
  ) {
    return false;
  }

  return true;
};

// Check if there was a force push after the comment
const hasForcePushAfterComment = async (
  owner: string,
  repo: string,
  pr: any,
  commentTimestamp: string
): Promise<boolean> => {
  try {
    const events = await fetchPREvents(owner, repo, pr.number);
    const commentDate = new Date(commentTimestamp);

    // Look for force push events after the comment
    const forcePushes = events.filter(
      (event) =>
        event.event === "head_ref_force_pushed" &&
        new Date(event.created_at) > commentDate
    );

    return forcePushes.length > 0;
  } catch (error: any) {
    console.error(
      `Error checking force pushes for PR #${pr.number}: ${error.message}`
    );
    return false;
  }
};

// Check if there were any commits after the comment
const hasCommitsAfterComment = async (
  owner: string,
  repo: string,
  pr: any,
  commentTimestamp: string
): Promise<boolean> => {
  try {
    const commits = await fetchPRCommits(owner, repo, pr.number);
    const commentDate = new Date(commentTimestamp);

    // Look for commits after the comment
    const laterCommits = commits.filter(
      (commit) => new Date(commit.commit.committer.date) > commentDate
    );

    return laterCommits.length > 0;
  } catch (error: any) {
    console.error(
      `Error checking commits for PR #${pr.number}: ${error.message}`
    );
    return false;
  }
};

// Process a single PR
const processPR = async (
  owner: string,
  repo: string,
  pr: any,
  scannedPRs: Set<string>
): Promise<Bug | null> => {
  const prNumber = pr.number;
  const prKey = `${owner}/${repo}#${prNumber}`;

  if (scannedPRs.has(prKey)) {
    console.log(
      `Skipping already scanned PR #${prNumber} in ${owner}/${repo}.`
    );
    return null;
  }

  if (pr.state === "open") {
    console.log(`Skipping open PR #${prNumber} in ${owner}/${repo}.`);
    scannedPRs.add(prKey);
    await saveSinglePR(prKey);
    return null;
  }

  console.log(
    `Processing ${
      pr.merged_at ? "merged" : "closed"
    } PR #${prNumber} in ${owner}/${repo}...`
  );

  try {
    // Only fetch review comments since we want line-specific comments
    const reviewComments = await fetchReviewComments(owner, repo, prNumber);

    let result: Bug | null = null;

    // Filter out comments that are replies or markdown-only changes or self-comments
    const nonReplyReviewComments = reviewComments.filter(
      (comment) =>
        !comment.in_reply_to_id &&
        comment.path &&
        comment.position &&
        comment.diff_hunk &&
        !comment.path.toLowerCase().endsWith(".md") && // Skip markdown files
        comment.user.login !== pr.user?.login // Skip self-comments
    );

    for (const comment of nonReplyReviewComments) {
      if (noLinksOrImages(comment)) {
        // Check for force pushes after this comment
        const hadForcePush = await hasForcePushAfterComment(
          owner,
          repo,
          pr,
          comment.created_at
        );
        if (hadForcePush) {
          console.log(
            `Skipping PR #${prNumber} due to force push after comment`
          );
          continue;
        }

        // Only check for later commits if PR was merged
        let hadLaterCommits = false;
        if (pr.merged_at) {
          hadLaterCommits = await hasCommitsAfterComment(
            owner,
            repo,
            pr,
            comment.created_at
          );
          if (hadLaterCommits) {
            console.log(
              `Skipping PR #${prNumber} as comment was made after commits and PR was merged`
            );
            continue;
          }
        }

        const reactions = await fetchCommentReactions(owner, repo, comment, HEADERS);
        const reactionCount = reactions.filter((r) =>
          ALLOWED_REACTIONS.includes(r.content)
        ).length;

        console.log(
          `Found a bug-related line comment in PR #${prNumber} in ${owner}/${repo}.`
        );
        result = {
          repo: `${owner}/${repo}`,
          prNumber,
          prTitle: pr.title, // Use title from PR list response
          prAuthor: pr.user?.login, // Use author from PR list response
          prLink: pr.html_url,
          commentAuthor: comment.user.login,
          commentBody: comment.body,
          commentBodyIncludesCodeSnippet: comment.body.includes("```"),
          commentLink: comment.html_url,
          filePath: comment.path,
          diffHunk: comment.diff_hunk,
          line: comment.position,
          reactionCount,
          reactions, // Store the full reactions data
          timestamp: comment.created_at,
          isReviewComment: true,
          pullRequestReviewId: comment.pull_request_review_id,
        };
        
        // Review and save result immediately when found, but only if it passes
        console.log(`Reviewing bug in PR #${prNumber} from ${owner}/${repo}...`);
        const reviewResult = await reviewBug(result);
        
        if (reviewResult !== null) {
          result.LLMReviewed = true;
          result.LLMReviewPassed = reviewResult;
          console.log(
            `Review result for PR #${prNumber}: ${
              reviewResult ? "PASS" : "FAIL"
            }`
          );
          
          // Only save and return bugs that pass the review
          if (reviewResult) {
            console.log(`Assessing difficulty for PR #${prNumber}...`);
            const difficultyLevel = await assessBugDifficulty(result);
            if (difficultyLevel !== null) {
              result.difficultyLevel = difficultyLevel;
              console.log(`Difficulty assessment result for PR #${prNumber}: ${difficultyLevel}`);
            } else {
              console.log(`Difficulty assessment failed for PR #${prNumber} (error occurred)`);
            }
            
            // Save the passing bug
            await appendToOutputFile([result]);
            console.log(`Saved PASSING bug-related comment to ${OUTPUT_FILE}`);
          } else {
            console.log(`Bug in PR #${prNumber} FAILED review and will NOT be saved.`);
            result = null; // Don't return failed reviews
          }
        } else {
          console.log(`Review failed for PR #${prNumber} (error occurred)`);
          result = null; // Don't return failed reviews
        }
        
        break;
      }
    }

    scannedPRs.add(prKey);
    await saveSinglePR(prKey);
    return result;
  } catch (error: any) {
    console.error(`Error processing PR #${prNumber}: ${error.message}`);
    return null;
  }
};

// Process a repository
const processRepo = async (
  repo: { owner: string; repo: string },
  scannedPRs: Set<string>,
  totalBugsFound: number
): Promise<number> => {
  const { owner, repo: repoName } = repo;
  console.log(`Processing repository: ${owner}/${repoName}`);
  let repoBugsFound = 0;

  const oldestPRNumber = findOldestScannedPR(scannedPRs, owner, repoName);
  if (oldestPRNumber > 0) {
    console.log(
      `Continuing from PR #${oldestPRNumber} in ${owner}/${repoName}`
    );
  }

  let page = 1;
  while (repoBugsFound < PER_REPO_MAX_BUGS) {
    const prs = await fetchPullRequests(owner, repoName, page);
    if (!prs.length) {
      console.log(
        `No more PRs found for ${owner}/${repoName} on page ${page}. Moving to next repository.`
      );
      break;
    }

    for (const pr of prs) {
      // Skip PRs with numbers higher than our target for React
      if (
        owner === "facebook" &&
        repoName === "react" &&
        pr.number > oldestPRNumber
      ) {
        // console.log(
        //   `Skipping PR #${pr.number} as it's higher than our target of #${oldestPRNumber}`
        // );
        continue;
      }

      const prKey = `${owner}/${repoName}#${pr.number}`;
      if (scannedPRs.has(prKey)) {
        continue;
      }

      const result = await processPR(owner, repoName, pr, scannedPRs);
      if (result) {
        // Only count bugs that passed the review (which is guaranteed by processPR now)
        repoBugsFound += 1;
        totalBugsFound += 1;
        console.log(
          `Found ${repoBugsFound}/${PER_REPO_MAX_BUGS} passing bugs in ${owner}/${repoName}`
        );
      }

      if (repoBugsFound >= PER_REPO_MAX_BUGS) {
        console.log(
          `Reached target of ${PER_REPO_MAX_BUGS} passing bugs for ${owner}/${repoName}. Moving to next repository.`
        );
        return totalBugsFound;
      }
    }

    page += 1;
  }

  return totalBugsFound;
};

// Main function
(async () => {
  const scannedPRs = await loadScannedPRs();
  let totalBugsFound = 0;

  for (const repo of REPOS) {
    totalBugsFound = await processRepo(repo, scannedPRs, totalBugsFound);
    console.log(
      `Completed repository ${repo.owner}/${repo.repo}. Total passing bugs found across all repos: ${totalBugsFound}`
    );
  }

  await saveScannedPRs(scannedPRs);
  console.log(`Saved scanned PRs to ${SCANNED_FILE}`);
})();