Add scraping logic and PostgreSQL/S3 integration

.github/workflows/deploy.yml

@@ -11,6 +11,12 @@ jobs:
       SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
       HOST: ${{ secrets.SERVER_HOST }}
       USER: ${{ secrets.SERVER_USER }}
+      DB_HOST: ${{ secrets.DB_HOST }}
+      DB_NAME: ${{ secrets.DB_NAME }}
+      DB_USER: ${{ secrets.DB_USER }}
+      DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/test-secrets.yml

@@ -0,0 +1,24 @@
+name: Test GitHub Secrets
+
+on:
+  workflow_dispatch: # permite rodar manualmente
+
+jobs:
+  test-secrets:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v3
+
+    - name: Print DB secrets
+      run: |
+        echo "DB_HOST = ${{ secrets.DB_HOST }}"
+        echo "DB_NAME = ${{ secrets.DB_NAME }}"
+        echo "DB_USER = ${{ secrets.DB_USER }}"
+        echo "DB_PASSWORD = ${{ secrets.DB_PASSWORD }}"
+
+    - name: Print AWS secrets
+      run: |
+        echo "AWS_ACCESS_KEY_ID = ${{ secrets.AWS_ACCESS_KEY_ID }}"
+        echo "AWS_SECRET_ACCESS_KEY = ${{ secrets.AWS_SECRET_ACCESS_KEY }}"

README.md

@@ -1,36 +1,37 @@
-This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
+# 💻 Software Engineering Internship (TechX Labs, Inc.)
 
-## Getting Started
+📅 **Duration:** September 08, 2025 – November 07, 2025  
+🖥️ **Location:** Remote (Boston, MA)
 
-First, run the development server:
+---
 
-```bash
-npm run dev
-# or
-yarn dev
-# or
-pnpm dev
-# or
-bun dev
-```
+## 📝 Internship Overview
 
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+Participated in the **Software Engineering Intern** program at TechX Labs, gaining hands-on experience with real-world projects focused on **large-scale web architecture and distributed systems**. Developed technical skills under the guidance of experienced engineers while contributing to enterprise-grade solutions.
 
-You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+---
 
-This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
+## ⚙️ Key Activities
 
-## Learn More
+🧪 Assisted in building and optimizing cloud-native infrastructure  
+🕵️‍♂️ Contributed to high-availability system design and performance improvements  
+🔍 Applied security best practices and worked with monitoring/logging systems  
+🧬 Collaborated with mentors and peers on real-world engineering projects  
+🛠️ Gained practical experience with technologies such as **VMSS, load balancers, and Redis caching**
 
-To learn more about Next.js, take a look at the following resources:
+---
 
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+## 💡 Skills Demonstrated
 
-You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
+🛡️ Cloud and Distributed Systems  
+🚨 Performance Optimization & High-Availability Design  
+📊 Enterprise-Grade Software Development  
+🌐 Security Best Practices Implementation  
+📁 Technical Documentation & Collaboration  
+🧠 Critical Thinking in Software Engineering
 
-## Deploy on Vercel
+---
 
-The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+## 🏁 Conclusion
 
-Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
+This internship provided me with valuable practical experience in software engineering, particularly in large-scale web architecture and distributed systems. It strengthened my technical and problem-solving skills while giving me confidence to contribute effectively to professional engineering teams. I look forward to applying these skills in future projects and advancing my knowledge in software development and cloud technologies.

app/api/conversation/[id]/route.ts

@@ -5,10 +5,6 @@ import { dbClient } from '@/lib/db/client';
 import { loadConfig } from '@/lib/config';
 
 let isInitialized = false;
-
-/**
- * Initialize services if not already initialized
- */
 async function ensureInitialized() {
   if (!isInitialized) {
     try {
@@ -17,7 +13,6 @@ async function ensureInitialized() {
       s3Client.initialize(config.s3);
       isInitialized = true;
     } catch (error) {
-      // If S3 client is already initialized, that's fine
       if (error instanceof Error && error.message.includes('already initialized')) {
         isInitialized = true;
       } else {
@@ -27,44 +22,14 @@ async function ensureInitialized() {
   }
 }
 
-/**
- * GET /api/conversation/[id]
- *
- * Retrieves the full conversation data including content and metadata
- *
- * @param request - The incoming request
- * @param context - Route context containing the conversation ID
- *
- * Response:
- * - 200: { conversation: ConversationRecord, content: string } - The conversation data and content
- * - 404: { error: string } - Conversation not found
- * - 500: { error: string } - Server error
- */
-export async function GET(
-  request: NextRequest,
-  { params }: { params: Promise<{ id: string }> }
-): Promise<NextResponse> {
+export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }): Promise<NextResponse> {
   try {
     await ensureInitialized();
     const id = (await params).id;
-
-    // Get conversation record from database
     const record = await getConversationRecord(id);
-
-    // Get conversation content from S3
     const content = await s3Client.getConversationContent(record.contentKey);
-
-    return NextResponse.json({
-      conversation: record,
-      content: content,
-    });
+    return NextResponse.json({ conversation: record, content });
   } catch (error) {
-    console.error('Error retrieving conversation:', error);
-
-    if (error instanceof Error && error.message.includes('not found')) {
-      return NextResponse.json({ error: error.message }, { status: 404 });
-    }
-
     return NextResponse.json({ error: 'Internal error, see logs' }, { status: 500 });
   }
 }

app/api/conversation/route.ts

@@ -1,26 +1,16 @@
 import { NextRequest, NextResponse } from 'next/server';
-import { parseHtmlToConversation } from '@/lib/parsers';
+import { getConversations } from '@/lib/db/conversations';
 import { dbClient } from '@/lib/db/client';
-import { s3Client } from '@/lib/storage/s3';
-import { CreateConversationInput } from '@/lib/db/types';
-import { createConversationRecord, getAllConversationRecords } from '@/lib/db/conversations';
-import { randomUUID } from 'crypto';
 import { loadConfig } from '@/lib/config';
 
 let isInitialized = false;
-
-/**
- * Initialize services if not already initialized
- */
 async function ensureInitialized() {
   if (!isInitialized) {
     try {
       const config = loadConfig();
       await dbClient.initialize(config.database);
-      s3Client.initialize(config.s3);
       isInitialized = true;
     } catch (error) {
-      // If S3 client is already initialized, that's fine
       if (error instanceof Error && error.message.includes('already initialized')) {
         isInitialized = true;
       } else {
@@ -30,136 +20,12 @@ async function ensureInitialized() {
   }
 }
 
-const ALLOWED_ORIGIN = '*';
-
-export async function OPTIONS() {
-  // Preflight handler
-  return new NextResponse(null, {
-    status: 204,
-    headers: {
-      'Access-Control-Allow-Origin': ALLOWED_ORIGIN,
-      'Access-Control-Allow-Methods': 'POST, GET, OPTIONS',
-      'Access-Control-Allow-Headers': 'Content-Type',
-    },
-  });
-}
-
-/**
- * POST /api/conversation
- *
- * Handles storing a new conversation from HTML input
- *
- * Request body (multipart/form-data):
- * - htmlDoc: File - The HTML document containing the conversation
- * - model: string - The AI model used (e.g., "ChatGPT", "Claude")
- *
- * Response:
- * - 201: { url: string } - The permalink URL for the conversation
- * - 400: { error: string } - Invalid request
- * - 500: { error: string } - Server error
- */
-export async function POST(req: NextRequest) {
-  try {
-    // Initialize services on first request
-    await ensureInitialized();
-
-    const formData = await req.formData();
-    const file = formData.get('htmlDoc');
-    const model = formData.get('model')?.toString() ?? 'ChatGPT';
-
-    // Validate input
-    if (!(file instanceof Blob)) {
-      return NextResponse.json({ error: '`htmlDoc` must be a file field' }, { status: 400 });
-    }
-
-    // Parse the conversation from HTML
-    const html = await file.text();
-    const conversation = await parseHtmlToConversation(html, model);
-
-    // Generate a unique ID for the conversation
-    const conversationId = randomUUID();
-
-    // Store only the conversation content in S3
-    const contentKey = await s3Client.storeConversation(conversationId, conversation.content);
-
-    // Create the database record with metadata
-    const dbInput: CreateConversationInput = {
-      model: conversation.model,
-      scrapedAt: new Date(conversation.scrapedAt),
-      sourceHtmlBytes: conversation.sourceHtmlBytes,
-      views: 0,
-      contentKey,
-    };
-
-    const record = await createConversationRecord(dbInput);
-
-    // Generate the permalink using the database-generated ID
-    const permalink = `${process.env.NEXT_PUBLIC_BASE_URL}/conversation/${record.id}`;
-
-    return NextResponse.json(
-      { url: permalink },
-      {
-        status: 201,
-        headers: {
-          'Access-Control-Allow-Origin': ALLOWED_ORIGIN,
-        },
-      }
-    );
-  } catch (err) {
-    console.error('Error processing conversation:', err);
-    return NextResponse.json({ error: 'Internal error, see logs' }, { status: 500 });
-  }
-}
-
-/**
- * GET /api/conversation
- *
- * Retrieves a list of all conversations with pagination
- *
- * Query parameters:
- * - limit: number (optional) - Maximum number of records to return (default: 50)
- * - offset: number (optional) - Number of records to skip (default: 0)
- *
- * Response:
- * - 200: { conversations: ConversationRecord[] } - Array of conversation records
- * - 400: { error: string } - Invalid request parameters
- * - 500: { error: string } - Server error
- */
-export async function GET(req: NextRequest) {
+export async function GET(request: NextRequest): Promise<NextResponse> {
   try {
-    // Initialize services on first request
     await ensureInitialized();
-
-    const { searchParams } = new URL(req.url);
-    const limitParam = searchParams.get('limit');
-    const offsetParam = searchParams.get('offset');
-
-    // Parse and validate query parameters
-    const limit = limitParam ? parseInt(limitParam, 10) : 50;
-    const offset = offsetParam ? parseInt(offsetParam, 10) : 0;
-
-    if (isNaN(limit) || limit < 1 || limit > 100) {
-      return NextResponse.json({ error: 'Invalid limit parameter. Must be between 1 and 100.' }, { status: 400 });
-    }
-
-    if (isNaN(offset) || offset < 0) {
-      return NextResponse.json({ error: 'Invalid offset parameter. Must be non-negative.' }, { status: 400 });
-    }
-
-    // Retrieve conversations from database
-    const conversations = await getAllConversationRecords(limit, offset);
-
-    return NextResponse.json(
-      { conversations },
-      {
-        status: 200,
-        headers: {
-          'Access-Control-Allow-Origin': ALLOWED_ORIGIN,
-        },
-      }
-    );
-  } catch (err) {
-    console.error('Error retrieving conversations:', err);
+    const conversations = await getConversations(50);
+    return NextResponse.json({ conversations });
+  } catch (error) {
     return NextResponse.json({ error: 'Internal error, see logs' }, { status: 500 });
   }
 }