From 73020236504b5df07d0f84f2164da55387107b97 Mon Sep 17 00:00:00 2001
From: blendi-remade <blendi@remade.ai>
Date: Tue, 21 Oct 2025 17:00:30 -0700
Subject: [PATCH 1/2] feat: implement background removal feature with API
 integration and UI support in timeline

---
 .../src/app/api/ai/remove-background/route.ts | 44 +++++++++++++++
 .../editor/timeline/timeline-element.tsx      | 54 +++++++++++++++++++
 apps/web/src/lib/fal-client.ts                | 41 +++++++++++++-
 apps/web/src/types/ai.ts                      | 10 ++++
 4 files changed, 148 insertions(+), 1 deletion(-)
 create mode 100644 apps/web/src/app/api/ai/remove-background/route.ts

diff --git a/apps/web/src/app/api/ai/remove-background/route.ts b/apps/web/src/app/api/ai/remove-background/route.ts
new file mode 100644
index 0000000..38a703e
--- /dev/null
+++ b/apps/web/src/app/api/ai/remove-background/route.ts
@@ -0,0 +1,44 @@
+import { NextResponse } from "next/server";
+import { fal } from "@fal-ai/client";
+import { z } from "zod";
+
+const requestSchema = z.object({
+  image_url: z.string().url(),
+});
+
+export async function POST(request: Request) {
+  try {
+    const body = await request.json();
+    const { image_url } = requestSchema.parse(body);
+
+    const result = await fal.subscribe("fal-ai/bria/background/remove", {
+      input: {
+        image_url,
+        sync_mode: true,
+      },
+      logs: true,
+      onQueueUpdate: (update) => {
+        if (update.status === "IN_PROGRESS") {
+          update.logs.map((log) => log.message).forEach(console.log);
+        }
+      },
+    });
+
+    return NextResponse.json(result.data);
+  } catch (error) {
+    console.error("Background removal error:", error);
+    
+    if (error instanceof z.ZodError) {
+      return NextResponse.json(
+        { error: "Invalid request parameters", details: error.errors },
+        { status: 400 }
+      );
+    }
+
+    return NextResponse.json(
+      { error: "Failed to remove background" },
+      { status: 500 }
+    );
+  }
+}
+
diff --git a/apps/web/src/components/editor/timeline/timeline-element.tsx b/apps/web/src/components/editor/timeline/timeline-element.tsx
index f9ae2f4..18c65ec 100644
--- a/apps/web/src/components/editor/timeline/timeline-element.tsx
+++ b/apps/web/src/components/editor/timeline/timeline-element.tsx
@@ -12,6 +12,7 @@ import {
   VolumeX,
   Sparkles,
   Video,
+  Eraser,
 } from "lucide-react";
 import { useMediaStore } from "@/stores/media-store";
 import { useTimelineStore } from "@/stores/timeline-store";
@@ -214,6 +215,55 @@ export function TimelineElement({
     });
   };
 
+  const handleRemoveBackground = async (e: React.MouseEvent) => {
+    e.stopPropagation();
+    
+    // Check if element is an image
+    if (element.type !== "media") return;
+    
+    const mediaItem = mediaFiles.find((file) => file.id === element.mediaId);
+    if (!mediaItem || mediaItem.type !== "image" || !mediaItem.url) return;
+
+    // Show processing toast
+    const toastId = toast.loading("Removing background...");
+
+    try {
+      // Dynamically import the removeBackground function
+      const { removeBackground } = await import("@/lib/fal-client");
+      
+      // Call the background removal API
+      const result = await removeBackground({ image_url: mediaItem.url });
+      
+      // Fetch the result image and create a blob URL
+      const response = await fetch(result.image.url);
+      const blob = await response.blob();
+      const newBlobUrl = URL.createObjectURL(blob);
+
+      // Update the media store with the new image
+      const { mediaFiles } = useMediaStore.getState();
+      useMediaStore.setState({
+        mediaFiles: mediaFiles.map(file => 
+          file.id === mediaItem.id 
+            ? { ...file, url: newBlobUrl, name: `${file.name}-no-bg` }
+            : file
+        )
+      });
+
+      // Clean up the old blob URL if it was a blob
+      if (mediaItem.url.startsWith('blob:')) {
+        URL.revokeObjectURL(mediaItem.url);
+      }
+
+      toast.success("Background removed successfully", { id: toastId });
+    } catch (error) {
+      console.error("Background removal error:", error);
+      toast.error("Failed to remove background", {
+        id: toastId,
+        description: error instanceof Error ? error.message : "Please try again",
+      });
+    }
+  };
+
   const renderElementContent = () => {
     if (element.type === "text") {
       return (
@@ -437,6 +487,10 @@ export function TimelineElement({
                   <Video className="h-4 w-4 mr-2" />
                   Animate with AI
                 </ContextMenuItem>
+                <ContextMenuItem onClick={handleRemoveBackground}>
+                  <Eraser className="h-4 w-4 mr-2" />
+                  Remove Background
+                </ContextMenuItem>
               </>
             )}
           </>
diff --git a/apps/web/src/lib/fal-client.ts b/apps/web/src/lib/fal-client.ts
index 4bfa009..97e8834 100644
--- a/apps/web/src/lib/fal-client.ts
+++ b/apps/web/src/lib/fal-client.ts
@@ -1,4 +1,4 @@
-import type { AIGenerationParams, AIGenerationResult, VideoGenerationParams, VideoGenerationResult } from "@/types/ai";
+import type { AIGenerationParams, AIGenerationResult, VideoGenerationParams, VideoGenerationResult, BackgroundRemovalParams, BackgroundRemovalResult } from "@/types/ai";
 
 /**
  * Convert blob URL to base64 data URI
@@ -118,4 +118,43 @@ export async function generateVideo({
       error instanceof Error ? error.message : "Failed to generate video. Please try again."
     );
   }
+}
+
+/**
+ * Remove background from an image using fal.ai's Bria RMBG 2.0 model
+ * Makes a server-side API call to protect the API key
+ */
+export async function removeBackground({
+  image_url,
+}: BackgroundRemovalParams): Promise<BackgroundRemovalResult> {
+  try {
+    // Convert blob URL to base64 data URI if needed
+    let processedImageUrl = image_url;
+    if (image_url.startsWith('blob:')) {
+      processedImageUrl = await blobUrlToDataUri(image_url);
+    }
+
+    const response = await fetch("/api/ai/remove-background", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        image_url: processedImageUrl,
+      }),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.json();
+      throw new Error(errorData.error || "Background removal failed");
+    }
+
+    const data = await response.json();
+    return data as BackgroundRemovalResult;
+  } catch (error) {
+    console.error("Background removal error:", error);
+    throw new Error(
+      error instanceof Error ? error.message : "Failed to remove background. Please try again."
+    );
+  }
 }
\ No newline at end of file
diff --git a/apps/web/src/types/ai.ts b/apps/web/src/types/ai.ts
index 5a5d14e..b3326f2 100644
--- a/apps/web/src/types/ai.ts
+++ b/apps/web/src/types/ai.ts
@@ -62,4 +62,14 @@ export interface VideoGenerationHistoryItem {
   params: VideoGenerationParams;
   result: VideoGenerationResult;
   timestamp: number;
+}
+
+// Background removal types
+export interface BackgroundRemovalParams {
+  image_url: string;
+  sync_mode?: boolean;
+}
+
+export interface BackgroundRemovalResult {
+  image: AIGeneratedImage;
 }
\ No newline at end of file

From 056d3ca32e605c2bd9236ed32d21596875442558 Mon Sep 17 00:00:00 2001
From: blendi-remade <blendi@remade.ai>
Date: Tue, 21 Oct 2025 18:17:33 -0700
Subject: [PATCH 2/2] feat: add video background removal API and integrate with
 timeline UI for seamless user experience

---
 .../api/ai/remove-video-background/route.ts   |  47 +++++++
 .../editor/timeline/timeline-element.tsx      | 125 +++++++++++++++++-
 apps/web/src/lib/fal-client.ts                |  45 ++++++-
 apps/web/src/types/ai.ts                      |  10 ++
 4 files changed, 222 insertions(+), 5 deletions(-)
 create mode 100644 apps/web/src/app/api/ai/remove-video-background/route.ts

diff --git a/apps/web/src/app/api/ai/remove-video-background/route.ts b/apps/web/src/app/api/ai/remove-video-background/route.ts
new file mode 100644
index 0000000..51d72de
--- /dev/null
+++ b/apps/web/src/app/api/ai/remove-video-background/route.ts
@@ -0,0 +1,47 @@
+import { NextResponse } from "next/server";
+import { fal } from "@fal-ai/client";
+import { z } from "zod";
+
+const requestSchema = z.object({
+  video_url: z.string().url(),
+  background_color: z.enum(["Transparent", "Black", "White", "Gray", "Red", "Green", "Blue", "Yellow", "Cyan", "Magenta", "Orange"]).optional(),
+  output_container_and_codec: z.enum(["mp4_h265", "mp4_h264", "webm_vp9", "mov_h265", "mov_proresks", "mkv_h265", "mkv_h264", "mkv_vp9", "gif"]).optional(),
+});
+
+export async function POST(request: Request) {
+  try {
+    const body = await request.json();
+    const { video_url, background_color, output_container_and_codec } = requestSchema.parse(body);
+
+    const result = await fal.subscribe("bria/video/background-removal", {
+      input: {
+        video_url,
+        background_color: background_color || "Black",
+        output_container_and_codec: output_container_and_codec || "webm_vp9",
+      },
+      logs: true,
+      onQueueUpdate: (update) => {
+        if (update.status === "IN_PROGRESS") {
+          update.logs.map((log) => log.message).forEach(console.log);
+        }
+      },
+    });
+
+    return NextResponse.json(result.data);
+  } catch (error) {
+    console.error("Video background removal error:", error);
+    
+    if (error instanceof z.ZodError) {
+      return NextResponse.json(
+        { error: "Invalid request parameters", details: error.errors },
+        { status: 400 }
+      );
+    }
+
+    return NextResponse.json(
+      { error: "Failed to remove video background" },
+      { status: 500 }
+    );
+  }
+}
+
diff --git a/apps/web/src/components/editor/timeline/timeline-element.tsx b/apps/web/src/components/editor/timeline/timeline-element.tsx
index 18c65ec..18cdeef 100644
--- a/apps/web/src/components/editor/timeline/timeline-element.tsx
+++ b/apps/web/src/components/editor/timeline/timeline-element.tsx
@@ -241,14 +241,37 @@ export function TimelineElement({
 
       // Update the media store with the new image
       const { mediaFiles } = useMediaStore.getState();
+      
+      // Convert blob to File for storage
+      const imageBlob = await fetch(newBlobUrl).then(r => r.blob());
+      const imageFile = new File([imageBlob], `${mediaItem.name}-no-bg.png`, {
+        type: "image/png",
+      });
+      
+      const updatedFile = {
+        ...mediaItem,
+        file: imageFile,
+        url: newBlobUrl,
+        name: `${mediaItem.name}-no-bg`,
+      };
+      
       useMediaStore.setState({
-        mediaFiles: mediaFiles.map(file => 
-          file.id === mediaItem.id 
-            ? { ...file, url: newBlobUrl, name: `${file.name}-no-bg` }
-            : file
+        mediaFiles: mediaFiles.map(f => 
+          f.id === mediaItem.id ? updatedFile : f
         )
       });
 
+      // Persist to storage
+      const { useProjectStore } = await import("@/stores/project-store");
+      const { storageService } = await import("@/lib/storage/storage-service");
+      const { activeProject } = useProjectStore.getState();
+      if (activeProject) {
+        await storageService.saveMediaFile({
+          projectId: activeProject.id,
+          mediaItem: updatedFile,
+        });
+      }
+
       // Clean up the old blob URL if it was a blob
       if (mediaItem.url.startsWith('blob:')) {
         URL.revokeObjectURL(mediaItem.url);
@@ -264,6 +287,94 @@ export function TimelineElement({
     }
   };
 
+  const handleRemoveVideoBackground = async (e: React.MouseEvent) => {
+    e.stopPropagation();
+    
+    // Check if element is a video
+    if (element.type !== "media") return;
+    
+    const mediaItem = mediaFiles.find((file) => file.id === element.mediaId);
+    if (!mediaItem || mediaItem.type !== "video" || !mediaItem.url) return;
+
+    // Show processing toast
+    const toastId = toast.loading("Removing video background... This may take a minute");
+
+    try {
+      // Dynamically import the removeVideoBackground function
+      const { removeVideoBackground } = await import("@/lib/fal-client");
+      
+      // Call the video background removal API with black background
+      // Note: Black works better for preview/export, can be keyed out later
+      const result = await removeVideoBackground({ 
+        video_url: mediaItem.url,
+        background_color: "Black",
+        output_container_and_codec: "webm_vp9",
+      });
+      
+      // Use the fal.ai URL directly - it has proper headers/codec info
+      // Fetching strips the content-type, so we'll fetch for File storage but use direct URL
+      const newVideoUrl = result.video.url;
+      
+      // Fetch blob for File storage
+      const response = await fetch(newVideoUrl);
+      const blob = await response.blob();
+      const file = new File([blob], result.video.file_name || `${mediaItem.name}-no-bg.webm`, {
+        type: "video/webm", // Type doesn't matter for File storage
+      });
+
+      // Generate thumbnail for the new video
+      const { generateVideoThumbnail } = await import("@/stores/media-store");
+      const { thumbnailUrl } = await generateVideoThumbnail(file);
+
+      // Clean up the old blob URLs FIRST
+      if (mediaItem.url.startsWith('blob:')) {
+        URL.revokeObjectURL(mediaItem.url);
+      }
+      if (mediaItem.thumbnailUrl?.startsWith('blob:')) {
+        URL.revokeObjectURL(mediaItem.thumbnailUrl);
+      }
+
+      // Update the media store with the new video (use direct fal.ai URL)
+      const { mediaFiles } = useMediaStore.getState();
+      const updatedFile = {
+        ...mediaItem,
+        file,
+        url: newVideoUrl, // Use fal.ai URL directly - has proper headers
+        thumbnailUrl,
+        name: `${mediaItem.name}-no-bg`,
+      };
+      
+      useMediaStore.setState({
+        mediaFiles: mediaFiles.map(f => 
+          f.id === mediaItem.id ? updatedFile : f
+        )
+      });
+
+      // Clear video cache AFTER updating state
+      const { videoCache } = await import("@/lib/video-cache");
+      videoCache.clearVideo(mediaItem.id);
+
+      // Persist to storage
+      const { useProjectStore } = await import("@/stores/project-store");
+      const { storageService } = await import("@/lib/storage/storage-service");
+      const { activeProject } = useProjectStore.getState();
+      if (activeProject) {
+        await storageService.saveMediaFile({
+          projectId: activeProject.id,
+          mediaItem: updatedFile,
+        });
+      }
+
+      toast.success("Video background removed successfully", { id: toastId });
+    } catch (error) {
+      console.error("Video background removal error:", error);
+      toast.error("Failed to remove video background", {
+        id: toastId,
+        description: error instanceof Error ? error.message : "Please try again",
+      });
+    }
+  };
+
   const renderElementContent = () => {
     if (element.type === "text") {
       return (
@@ -493,6 +604,12 @@ export function TimelineElement({
                 </ContextMenuItem>
               </>
             )}
+            {mediaItem?.type === "video" && (
+              <ContextMenuItem onClick={handleRemoveVideoBackground}>
+                <Eraser className="h-4 w-4 mr-2" />
+                Remove Background
+              </ContextMenuItem>
+            )}
           </>
         )}
 
diff --git a/apps/web/src/lib/fal-client.ts b/apps/web/src/lib/fal-client.ts
index 97e8834..1183100 100644
--- a/apps/web/src/lib/fal-client.ts
+++ b/apps/web/src/lib/fal-client.ts
@@ -1,4 +1,4 @@
-import type { AIGenerationParams, AIGenerationResult, VideoGenerationParams, VideoGenerationResult, BackgroundRemovalParams, BackgroundRemovalResult } from "@/types/ai";
+import type { AIGenerationParams, AIGenerationResult, VideoGenerationParams, VideoGenerationResult, BackgroundRemovalParams, BackgroundRemovalResult, VideoBackgroundRemovalParams, VideoBackgroundRemovalResult } from "@/types/ai";
 
 /**
  * Convert blob URL to base64 data URI
@@ -157,4 +157,47 @@ export async function removeBackground({
       error instanceof Error ? error.message : "Failed to remove background. Please try again."
     );
   }
+}
+
+/**
+ * Remove background from a video using fal.ai's Bria video background removal model
+ * Makes a server-side API call to protect the API key
+ */
+export async function removeVideoBackground({
+  video_url,
+  background_color = "Transparent",
+  output_container_and_codec = "webm_vp9",
+}: VideoBackgroundRemovalParams): Promise<VideoBackgroundRemovalResult> {
+  try {
+    // Convert blob URL to base64 data URI if needed
+    let processedVideoUrl = video_url;
+    if (video_url.startsWith('blob:')) {
+      processedVideoUrl = await blobUrlToDataUri(video_url);
+    }
+
+    const response = await fetch("/api/ai/remove-video-background", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        video_url: processedVideoUrl,
+        background_color,
+        output_container_and_codec,
+      }),
+    });
+
+    if (!response.ok) {
+      const errorData = await response.json();
+      throw new Error(errorData.error || "Video background removal failed");
+    }
+
+    const data = await response.json();
+    return data as VideoBackgroundRemovalResult;
+  } catch (error) {
+    console.error("Video background removal error:", error);
+    throw new Error(
+      error instanceof Error ? error.message : "Failed to remove video background. Please try again."
+    );
+  }
 }
\ No newline at end of file
diff --git a/apps/web/src/types/ai.ts b/apps/web/src/types/ai.ts
index b3326f2..b2fea76 100644
--- a/apps/web/src/types/ai.ts
+++ b/apps/web/src/types/ai.ts
@@ -72,4 +72,14 @@ export interface BackgroundRemovalParams {
 
 export interface BackgroundRemovalResult {
   image: AIGeneratedImage;
+}
+
+export interface VideoBackgroundRemovalParams {
+  video_url: string;
+  background_color?: "Transparent" | "Black" | "White" | "Gray" | "Red" | "Green" | "Blue" | "Yellow" | "Cyan" | "Magenta" | "Orange";
+  output_container_and_codec?: "mp4_h265" | "mp4_h264" | "webm_vp9" | "mov_h265" | "mov_proresks" | "mkv_h265" | "mkv_h264" | "mkv_vp9" | "gif";
+}
+
+export interface VideoBackgroundRemovalResult {
+  video: AIGeneratedVideo;
 }
\ No newline at end of file