review-utils-difficulty.ts

import axios from "axios";
import * as dotenv from "dotenv";
import fs from "fs";

// Load environment variables
dotenv.config();

const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;

if (!GITHUB_TOKEN) throw new Error("GITHUB_TOKEN is not set");
if (!OPENAI_API_KEY) throw new Error("OPENAI_API_KEY is not set");

const HEADERS = {
  Authorization: `Bearer ${GITHUB_TOKEN}`,
  Accept: "application/vnd.github+json",
};

const PROMPTS_FILE = "review-difficulty-prompts.txt";

// Helper function to add retry logic
async function fetchWithRetry<T>(
  fn: () => Promise<T>,
  retries = 3,
  delay = 1000
): Promise<T> {
  try {
    return await fn();
  } catch (error) {
    if (retries === 0) throw error;
    console.log(
      `Request failed, retrying in ${delay}ms... (${retries} retries left)`
    );
    await new Promise((resolve) => setTimeout(resolve, delay));
    return fetchWithRetry(fn, retries - 1, delay * 2);
  }
}

// Function to append prompt to file
async function savePrompt(prNumber: number, prompt: string) {
  const separator = "\n" + "=".repeat(100) + "\n";
  const header = `PR #${prNumber} - ${new Date().toISOString()}\n`;
  const content = separator + header + prompt + separator;
  await fs.promises.appendFile(PROMPTS_FILE, content);
}

export type DifficultyLevel = "easy" | "medium" | "hard";

// Main difficulty assessment function
export async function assessBugDifficulty(bug: any): Promise<DifficultyLevel | null> {
  try {
    const prompt = `You are assessing the difficulty of identifying a bug in a PR review comment.
The goal is to classify how difficult it would be for a typical software engineer to identify this issue by looking at the code.

Classify the difficulty as one of:
- EASY: The bug is obvious from looking at the code snippet and comment. Any reasonable software engineer would quickly spot the issue.
- MEDIUM: The bug requires some analysis of the code or basic knowledge of common programming concepts, but doesn't need deep system knowledge.
- HARD: The bug requires extensive context beyond the diff or deep domain knowledge. This includes framework-specific architectural rules (like import/dependency patterns), internal library organization requirements, or other conventions that aren't apparent without deep familiarity with the codebase.

Repository Name: ${bug.repo}
PR Title: ${bug.prTitle}

PR Review Comment:
@${bug.commentAuthor}: ${bug.commentBody}

Comment Context Code File Path: ${bug.filePath}
Comment Context Code Diff:
${bug.diffHunk}

Based on the criteria above, respond with only EASY, MEDIUM, or HARD.`;

    // Save prompt before sending to OpenAI
    await savePrompt(bug.prNumber, prompt);

    try {
      console.log(`  Sending request to OpenAI API for PR #${bug.prNumber}...`);
      
      const response = await axios.post(
        "https://api.openai.com/v1/chat/completions",
        {
          model: "gpt-4",
          messages: [{ role: "user", content: prompt }],
          temperature: 0,
        },
        {
          headers: {
            Authorization: `Bearer ${OPENAI_API_KEY}`,
            "Content-Type": "application/json",
          },
        }
      );

      console.log(`  Received response from OpenAI API`);
      
      const answer = response.data.choices[0].message.content.trim();
      console.log(`  Raw API response: "${answer}"`);
      
      switch (answer) {
        case "EASY":
          return "easy";
        case "MEDIUM":
          return "medium";
        case "HARD":
          return "hard";
        default:
          console.error(`  Unexpected response from OpenAI: "${answer}"`);
          // Try to extract a valid difficulty level from the response
          if (answer.includes("EASY")) return "easy";
          if (answer.includes("MEDIUM")) return "medium";
          if (answer.includes("HARD")) return "hard";
          return null;
      }
    } catch (error) {
      if (axios.isAxiosError(error)) {
        console.error(`  API Error: ${error.message}`);
        if (error.response) {
          console.error(`  Status: ${error.response.status}`);
          console.error(`  Data: ${JSON.stringify(error.response.data, null, 2)}`);
        }
      } else {
        console.error(`  Error calling OpenAI API: ${error}`);
      }
      return null;
    }
  } catch (error) {
    console.error(`  Error assessing bug difficulty: ${error}`);
    return null;
  }
}