implement bucket

.github/workflows/test-lint.yml

@@ -53,6 +53,13 @@ jobs:
         run: |
           deno install --allow-scripts --reload
 
+      - uses: fsouza/[email protected]
+        with:
+          version: "latest"
+          backend: memory
+          public-host: "localhost:4443"
+          scheme: http
+
       - name: Run migrations
         run: deno task db:migrate
 

deno.json

@@ -11,13 +11,14 @@
     "dev:runner": "deno task --config ./packages/runner/deno.json dev",
     "build:app": "deno task --config ./packages/app/deno.json build",
     "build:core": "deno task --config ./packages/core/deno.json build",
-    "test": "STRIPE_USE_MOCK=true deno test -A",
+    "test": "STRIPE_USE_MOCK=true GCP_USE_FAKE_GCS_SERVER=true deno test -A",
     "db:docker": "docker container rm stackcore-pg --force || true && docker run --name stackcore-pg -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -e POSTGRES_DB=core -p 5432:5432 -d postgres:17",
     "db:migrate": "deno task --config ./packages/core/deno.json migrate -A",
     "stripe:create-products": "deno run -A --env-file=.env ./packages/core/src/stripe/scripts/createProducts.ts",
     "stripe:cli": "docker run --rm -it --env-file=.env stripe/stripe-cli:latest",
     "stripe:forward": "deno task stripe:cli listen --forward-to http://host.docker.internal:4000/billing/webhook",
-    "stripe:mock": "docker run --rm -it -p 12111-12112:12111-12112 stripe/stripe-mock:latest"
+    "stripe:mock": "docker run --rm -it -p 12111-12112:12111-12112 stripe/stripe-mock:latest",
+    "bucket:mock": "docker run --rm -it -p 4443:4443 fsouza/fake-gcs-server:latest -scheme http -public-host localhost:4443"
   },
   "lint": {
     "exclude": [

packages/app/src/pages/projects/add.tsx

@@ -47,6 +47,7 @@ export default function AddProjectPage() {
 
   const formSchema = z.object({
     name: z.string().min(1, "Project name is required").max(100),
+    repoUrl: z.string().url("Invalid repository URL"),
     maxCodeCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
     maxCodeCharPerFile: z.number().int().min(1, "Must be at least 1"),
     maxCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
@@ -74,6 +75,7 @@ export default function AddProjectPage() {
     disabled: isBusy,
     defaultValues: {
       name: "",
+      repoUrl: "",
       maxCodeCharPerSymbol: 1000,
       maxCodeCharPerFile: 50000,
       maxCharPerSymbol: 2000,
@@ -102,6 +104,7 @@ export default function AddProjectPage() {
     try {
       const { url, method, body } = ProjectApiTypes.prepareCreateProject({
         name: values.name,
+        repoUrl: values.repoUrl,
         workspaceId: selectedWorkspaceId,
         maxCodeCharPerSymbol: values.maxCodeCharPerSymbol,
         maxCodeCharPerFile: values.maxCodeCharPerFile,
@@ -325,6 +328,28 @@ export default function AddProjectPage() {
                       </FormItem>
                     )}
                   />
+                  <FormField
+                    control={form.control}
+                    name="repoUrl"
+                    render={({ field }) => (
+                      <FormItem>
+                        <FormLabel>
+                          Repository URL{" "}
+                          <span className="text-destructive">*</span>
+                        </FormLabel>
+                        <FormControl>
+                          <Input
+                            {...field}
+                            placeholder="https://github.com/user/repo"
+                          />
+                        </FormControl>
+                        <FormDescription>
+                          The URL of the repository for this project
+                        </FormDescription>
+                        <FormMessage />
+                      </FormItem>
+                    )}
+                  />
                 </div>
 
                 {/* Configuration Tabs */}

packages/app/src/pages/projects/project/manifests/manifest.tsx

@@ -21,14 +21,25 @@ export default function ProjectManifest() {
     ManifestApiTypes.GetManifestDetailsResponse | undefined
   >(undefined);
 
-  const [_dependencyManifest, setDependencyManifest] = useState<
+  const [dependencyManifest, setDependencyManifest] = useState<
     DependencyManifest | undefined
   >(undefined);
 
   const [auditManifest, setAuditManifest] = useState<
     ManifestApiTypes.GetManifestAuditResponse | undefined
   >(undefined);
 
+  async function fetchManifest(url: string) {
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error("Failed to fetch manifest");
+    }
+    const manifest = await response
+      .json() as unknown as DependencyManifest;
+
+    return manifest;
+  }
+
   useEffect(() => {
     async function fetchData() {
       if (!manifestId) {
@@ -53,9 +64,12 @@ export default function ProjectManifest() {
           throw new Error("Failed to fetch manifest");
         }
 
-        const manifest = await manifestResponse.json();
+        const manifest = await manifestResponse
+          .json() as ManifestApiTypes.GetManifestDetailsResponse;
         setManifestData(manifest);
-        setDependencyManifest(manifest.manifest);
+
+        const dependencyManifest = await fetchManifest(manifest.manifest);
+        setDependencyManifest(dependencyManifest);
 
         // Fetch audit manifest
         const { url: auditUrl, method: auditMethod } = ManifestApiTypes
@@ -93,7 +107,7 @@ export default function ProjectManifest() {
   return (
     <DependencyVisualizer
       manifestId={manifestData?.id || 0}
-      dependencyManifest={manifestData?.manifest as DependencyManifest}
+      dependencyManifest={dependencyManifest as DependencyManifest}
       auditManifest={auditManifest as AuditManifest}
     />
   );

packages/app/src/pages/projects/project/settings.tsx

@@ -46,6 +46,7 @@ export default function ProjectSettings() {
 
   const formSchema = z.object({
     name: z.string().min(1, "Project name is required").max(100),
+    repoUrl: z.string().url("Invalid repository URL"),
     maxCodeCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
     maxCodeCharPerFile: z.number().int().min(1, "Must be at least 1"),
     maxCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
@@ -73,6 +74,7 @@ export default function ProjectSettings() {
     disabled: isBusy,
     defaultValues: {
       name: context.project.name,
+      repoUrl: context.project.repo_url,
       maxCodeCharPerSymbol: context.project.max_code_char_per_symbol,
       maxCodeCharPerFile: context.project.max_code_char_per_file,
       maxCharPerSymbol: context.project.max_char_per_symbol,
@@ -100,6 +102,7 @@ export default function ProjectSettings() {
         context.project.id,
         {
           name: values.name,
+          repoUrl: values.repoUrl,
           maxCodeCharPerSymbol: values.maxCodeCharPerSymbol,
           maxCodeCharPerFile: values.maxCodeCharPerFile,
           maxCharPerSymbol: values.maxCharPerSymbol,
@@ -298,6 +301,28 @@ export default function ProjectSettings() {
                     </FormItem>
                   )}
                 />
+                <FormField
+                  control={form.control}
+                  name="repoUrl"
+                  render={({ field }) => (
+                    <FormItem>
+                      <FormLabel>
+                        Repository URL{" "}
+                        <span className="text-destructive">*</span>
+                      </FormLabel>
+                      <FormControl>
+                        <Input
+                          {...field}
+                          placeholder="https://github.com/user/repo"
+                        />
+                      </FormControl>
+                      <FormDescription>
+                        The URL of the repository for this project
+                      </FormDescription>
+                      <FormMessage />
+                    </FormItem>
+                  )}
+                />
               </div>
 
               {/* Configuration Tabs */}
@@ -380,6 +405,7 @@ export default function ProjectSettings() {
                 </Button>
                 <Button
                   variant="outline"
+                  type="button"
                   onClick={() =>
                     navigate(`/projects/${context.project.id}/manifests`)}
                   disabled={isBusy}

packages/core/deno.json

@@ -7,6 +7,7 @@
     "build": "deno compile --allow-all --include src/db/migrations --output ./dist/core ./src/index.ts"
   },
   "imports": {
+    "@google-cloud/storage": "npm:@google-cloud/storage@^7.16.0",
     "@oak/oak": "jsr:@oak/oak@^17.1.4",
     "@sentry/deno": "npm:@sentry/deno@^9.28.1",
     "@std/assert": "jsr:@std/assert@1",

packages/core/src/api/manifest/router.test.ts

@@ -51,6 +51,7 @@ Deno.test("create a manifest", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -141,6 +142,7 @@ Deno.test("create a manifest - non-member of workspace", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -260,6 +262,7 @@ Deno.test("create manifest - workspace access disabled", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -338,6 +341,7 @@ Deno.test("get manifests", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -429,6 +433,7 @@ Deno.test("get manifests with workspace filter", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -513,6 +518,7 @@ Deno.test("get manifests with search", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -608,6 +614,7 @@ Deno.test("get manifests - pagination", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -736,6 +743,7 @@ Deno.test("get manifest details", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -776,6 +784,12 @@ Deno.test("get manifest details", async () => {
       throw new Error(createResponse.error);
     }
 
+    await db
+      .selectFrom("manifest")
+      .selectAll()
+      .where("id", "=", createResponse.id)
+      .executeTakeFirstOrThrow();
+
     const { url, method } = ManifestApiTypes.prepareGetManifestDetails(
       createResponse.id,
     );
@@ -796,8 +810,15 @@ Deno.test("get manifest details", async () => {
     assertEquals(manifest.branch, "main");
     assertEquals(manifest.commitSha, "abc123");
     assertEquals(manifest.version, 1);
-    assertEquals(manifest.manifest.test, "data");
-    assertEquals(manifest.manifest.complex.nested, "value");
+    assertEquals(manifest.manifest.length > 0, true);
+
+    // get the content of the manifest from the bucket
+    const manifestContentResponse = await fetch(
+      manifest.manifest,
+    );
+    const manifestContent = await manifestContentResponse.json();
+    assertEquals(manifestContent.test, "data");
+    assertEquals(manifestContent.complex.nested, "value");
   } finally {
     await resetTables();
     await destroyKyselyDb();
@@ -827,6 +848,7 @@ Deno.test("get manifest details - non-member", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -915,6 +937,7 @@ Deno.test("delete a manifest", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -1007,6 +1030,7 @@ Deno.test("delete a manifest - non-member", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -1104,6 +1128,7 @@ Deno.test("get manifest audit", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -1255,6 +1280,7 @@ Deno.test("get manifest audit - non-member", async () => {
       userId,
       {
         name: "Test Project",
+        repoUrl: "https://github.com/test/test",
         workspaceId: workspace.id,
         maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
         maxCodeCharPerFile: config.maxCodeCharPerFile,

packages/core/src/api/manifest/service.ts

@@ -8,6 +8,11 @@ import type {
   GetManifestDetailsResponse,
   GetManifestsResponse,
 } from "./types.ts";
+import {
+  downloadJsonFromBucket,
+  getPublicLink,
+  uploadJsonToBucket,
+} from "../../bucketStorage/index.ts";
 
 export const manifestNotFoundError = "manifest_not_found";
 export const projectNotFoundError = "project_not_found";
@@ -66,6 +71,9 @@ export class ManifestService {
       return { error: accessDisabledError };
     }
 
+    const manifestFileName = `${projectId}-${Date.now()}.json`;
+    await uploadJsonToBucket(manifest, manifestFileName);
+
     // Create the manifest
     const newManifest = await db
       .insertInto("manifest")
@@ -75,7 +83,7 @@ export class ManifestService {
         commitSha,
         commitShaDate,
         version: settings.MANIFEST.DEFAULT_VERSION,
-        manifest,
+        manifest: manifestFileName,
         created_at: new Date(),
       })
       .returningAll()
@@ -216,7 +224,12 @@ export class ManifestService {
       return { error: manifestNotFoundError };
     }
 
-    return manifest;
+    const publicLink = await getPublicLink(manifest.manifest);
+
+    return {
+      ...manifest,
+      manifest: publicLink,
+    };
   }
 
   /**
@@ -284,10 +297,14 @@ export class ManifestService {
       return { error: projectNotFoundError };
     }
 
+    const manifestJson = await downloadJsonFromBucket(
+      manifest.manifest,
+    ) as DependencyManifest;
+
     try {
       const auditManifest = generateAuditManifest(
         manifest.version,
-        manifest.manifest as DependencyManifest,
+        manifestJson,
         {
           file: {
             maxCodeChar: project.max_char_per_file,