BugDetectionBench/analyze-bugs.ts at main · moritzWa/BugDetectionBench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import fs from "fs";
import { promisify } from "util";
import { DifficultyLevel } from "./review-utils-difficulty";

const readFileAsync = promisify(fs.readFile);

const INPUT_FILE = "bugs.json";

interface Bug {
  repo: string;
  prNumber: number;
  prTitle: string;
  prLink: string;
  commentAuthor: string;
  commentBody: string;
  commentBodyIncludesCodeSnippet: boolean;
  commentLink: string;
  filePath: string;
  diffHunk: string;
  line: number;
  reactionCount: number;
  timestamp: string;
  isReviewComment: boolean;
  pullRequestReviewId: number;
  LLMReviewed?: boolean;
  LLMReviewPassed?: boolean;
  difficultyLevel?: DifficultyLevel;
  difficultyAssessedAt?: string;
}

async function analyzeBugs() {
  try {
    // Read the bugs file
    const data = await readFileAsync(INPUT_FILE, "utf8");
    const bugs = JSON.parse(data) as Bug[];

    const totalBugs = bugs.length;
    const reviewedBugs = bugs.filter((bug) => bug.LLMReviewed).length;
    const unreviewedBugs = totalBugs - reviewedBugs;
    const passedBugs = bugs.filter(
      (bug) => bug.LLMReviewed && bug.LLMReviewPassed
    ).length;
    const failedBugs = reviewedBugs - passedBugs;
    const passRate =
      reviewedBugs > 0 ? ((passedBugs / reviewedBugs) * 100).toFixed(2) : "0";

    // Group by repository
    const repoStats = new Map<
      string,
      { total: number; reviewed: number; passed: number }
    >();
    bugs.forEach((bug) => {
      const stats = repoStats.get(bug.repo) || {
        total: 0,
        reviewed: 0,
        passed: 0,
      };
      stats.total++;
      if (bug.LLMReviewed) {
        stats.reviewed++;
        if (bug.LLMReviewPassed) stats.passed++;
      }
      repoStats.set(bug.repo, stats);
    });

    // Count bugs by difficulty level
    const bugsWithDifficulty = bugs.filter(bug => bug.difficultyLevel);
    const easyBugs = bugs.filter(bug => bug.difficultyLevel === "easy").length;
    const mediumBugs = bugs.filter(bug => bug.difficultyLevel === "medium").length;
    const hardBugs = bugs.filter(bug => bug.difficultyLevel === "hard").length;
    const unassessedBugs = totalBugs - bugsWithDifficulty.length;

    console.log("\n=== Bug Review Statistics ===");
    console.log(`Total bugs found: ${totalBugs}`);
    console.log(`\nReview Progress:`);
    console.log(`- Reviewed: ${reviewedBugs}`);
    console.log(`- Not yet reviewed: ${unreviewedBugs}`);
    console.log(`\nReview Results:`);
    console.log(`- Passed: ${passedBugs}`);
    console.log(`- Failed: ${failedBugs}`);
    console.log(`- Pass rate: ${passRate}%`);

    console.log("\nDifficulty Breakdown:");
    console.log(`- Easy: ${easyBugs} (${((easyBugs / bugsWithDifficulty.length) * 100).toFixed(2)}%)`);
    console.log(`- Medium: ${mediumBugs} (${((mediumBugs / bugsWithDifficulty.length) * 100).toFixed(2)}%)`);
    console.log(`- Hard: ${hardBugs} (${((hardBugs / bugsWithDifficulty.length) * 100).toFixed(2)}%)`);
    console.log(`- Not assessed: ${unassessedBugs}`);

    // console.log("\nRepository Breakdown:");
    // repoStats.forEach((stats, repo) => {
    //   const repoPassRate =
    //     stats.reviewed > 0
    //       ? ((stats.passed / stats.reviewed) * 100).toFixed(2)
    //       : "0";
    //   console.log(`\n${repo}:`);
    //   console.log(`- Total bugs: ${stats.total}`);
    //   console.log(`- Reviewed: ${stats.reviewed}`);
    //   console.log(`- Passed: ${stats.passed}`);
    //   console.log(`- Pass rate: ${repoPassRate}%`);
    // });
  } catch (error) {
    console.error("Error analyzing bugs.json:", error);
  }
}

analyzeBugs();