From 9609bc5e068f6749bca91f64bf0c92783fca487d Mon Sep 17 00:00:00 2001 From: "Jennifer J. Sun" <40725770+jenjsun@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:12:41 -0700 Subject: [PATCH 1/5] Adding videoprism to model-libraries.ts --- packages/tasks/src/model-libraries.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 9641ed689c..c4342a244e 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -1086,6 +1086,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { countDownloads: `path_extension:"pth"`, snippets: snippets.sana, }, + "videoprism": { + prettyLabel: "VideoPrism", + repoName: "VideoPrism", + repoUrl: "https://github.com/google-deepmind/videoprism", + countDownloads: `path_extension:"npz"`, + snippets: snippets.videoprism, + }, "vfi-mamba": { prettyLabel: "VFIMamba", repoName: "VFIMamba", From feeca0de5901282fdca565e44cf3743ea99b7c51 Mon Sep 17 00:00:00 2001 From: "Jennifer J. Sun" <40725770+jenjsun@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:58:39 -0700 Subject: [PATCH 2/5] Update videoprism in model-libraries-snippets.ts --- packages/tasks/src/model-libraries-snippets.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index a5e63fae65..b930211211 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -1450,6 +1450,23 @@ image = sana( ) `, ]; +export const videoprism = (model: ModelData): string[] => [ + `# Install from https://github.com/google-deepmind/videoprism + import jax +import jax.numpy as jnp +from videoprism import models as vp + +# Models available: ['videoprism_public_v1_base', 'videoprism_public_v1_large'] +MODEL_NAME = 'videoprism_public_v1_base' + +flax_model = vp.MODELS[MODEL_NAME]() +loaded_state = vp.load_pretrained_weights_from_hf(MODEL_NAME) + +@jax.jit +def forward_fn(inputs, train=False): + return flax_model.apply(loaded_state, inputs, train=train)`, +]; + export const vfimamba = (model: ModelData): string[] => [ `from Trainer_finetune import Model From bf1bdc3ea61cf9c4fd3b2494acd2e34d0c2fc32c Mon Sep 17 00:00:00 2001 From: "Jennifer J. Sun" <40725770+jenjsun@users.noreply.github.com> Date: Mon, 30 Jun 2025 13:19:41 -0700 Subject: [PATCH 3/5] Update model-libraries-snippets.ts --- packages/tasks/src/model-libraries-snippets.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index b930211211..56e3d7e06e 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -1452,15 +1452,15 @@ image = sana( export const videoprism = (model: ModelData): string[] => [ `# Install from https://github.com/google-deepmind/videoprism - import jax +import jax import jax.numpy as jnp from videoprism import models as vp -# Models available: ['videoprism_public_v1_base', 'videoprism_public_v1_large'] -MODEL_NAME = 'videoprism_public_v1_base' +# Models available: ['videoprism_public_v1_base_hf', 'videoprism_public_v1_large_hf'] +MODEL_NAME = 'videoprism_public_v1_base_hf' flax_model = vp.MODELS[MODEL_NAME]() -loaded_state = vp.load_pretrained_weights_from_hf(MODEL_NAME) +loaded_state = vp.load_pretrained_weights(MODEL_NAME) @jax.jit def forward_fn(inputs, train=False): From eb1cde989cfd57acaf409eade241de6d84b5fa5f Mon Sep 17 00:00:00 2001 From: "Jennifer J. Sun" <40725770+jenjsun@users.noreply.github.com> Date: Wed, 9 Jul 2025 08:14:07 -0700 Subject: [PATCH 4/5] Update model-libraries-snippets.ts --- packages/tasks/src/model-libraries-snippets.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index b7949b894e..ba5e6a87df 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -1548,11 +1548,8 @@ import jax import jax.numpy as jnp from videoprism import models as vp -# Models available: ['videoprism_public_v1_base_hf', 'videoprism_public_v1_large_hf'] -MODEL_NAME = 'videoprism_public_v1_base_hf' - -flax_model = vp.MODELS[MODEL_NAME]() -loaded_state = vp.load_pretrained_weights(MODEL_NAME) +flax_model = vp.MODELS["${model.id}"]() +loaded_state = vp.load_pretrained_weights("${model.id}") @jax.jit def forward_fn(inputs, train=False): From bde079bdfb4c736c8d4eef89c691bec42f14d7f5 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Thu, 10 Jul 2025 09:13:27 +0200 Subject: [PATCH 5/5] Apply suggestions from code review --- packages/tasks/src/model-libraries.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 4f68c2c91e..7662413b21 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -1093,13 +1093,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { countDownloads: `path_extension:"pth"`, snippets: snippets.sana, }, - "videoprism": { + videoprism: { prettyLabel: "VideoPrism", repoName: "VideoPrism", repoUrl: "https://github.com/google-deepmind/videoprism", countDownloads: `path_extension:"npz"`, snippets: snippets.videoprism, - }, + }, "vfi-mamba": { prettyLabel: "VFIMamba", repoName: "VFIMamba",