From f8a96faab891dfc982ec1a618cfc544709a5710d Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 19 Jul 2026 23:37:37 +0400 Subject: [PATCH 1/8] chore(litertlm): migrate C-API patches + build refs to LiteRT-LM v0.14.0 (Phase 0) --- .../native/litert_lm/build_android.sh | 4 +- .../native/litert_lm/build_ios.sh | 4 +- .../native/litert_lm/build_macos.sh | 4 +- .../native/litert_lm/patch_c_api.sh | 479 ++---------------- 4 files changed, 36 insertions(+), 455 deletions(-) diff --git a/packages/flutter_gemma_litertlm/native/litert_lm/build_android.sh b/packages/flutter_gemma_litertlm/native/litert_lm/build_android.sh index e0ad7c8d..44d158d8 100755 --- a/packages/flutter_gemma_litertlm/native/litert_lm/build_android.sh +++ b/packages/flutter_gemma_litertlm/native/litert_lm/build_android.sh @@ -23,7 +23,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PREBUILT_DIR="$SCRIPT_DIR/prebuilt/android_arm64" LITERT_LM_DIR="/tmp/LiteRT-LM" -DEFAULT_REF="a0afb5a56acd106b23a2b2385b8469834dc268c0" +DEFAULT_REF="80f301ff9a3b02c2c1e7be2dd1a567752f7b51b6" VERSION="${1:-}" # Resolve Android NDK — prefer ANDROID_NDK_HOME env, else newest under @@ -68,7 +68,7 @@ git checkout -f "$TARGET_REF" echo "Building from: $(git log --oneline -1)" # 3. Ensure cc_binary(linkshared=True) target exists in c/BUILD -if ! grep -q "linkshared" c/BUILD; then +if ! grep -q '"libLiteRtLm.dylib"' c/BUILD; then cat >> c/BUILD << 'BUILDEOF' cc_binary( diff --git a/packages/flutter_gemma_litertlm/native/litert_lm/build_ios.sh b/packages/flutter_gemma_litertlm/native/litert_lm/build_ios.sh index efa694ce..d8fc5e63 100755 --- a/packages/flutter_gemma_litertlm/native/litert_lm/build_ios.sh +++ b/packages/flutter_gemma_litertlm/native/litert_lm/build_ios.sh @@ -49,14 +49,14 @@ fi # This is the first public LiteRT-LM commit where libLiteRtLm rebuilt from # source has matching ABI with the prebuilt accelerators. v0.11.0 itself # is broken — see the WARNING above and the upstream issue we filed. -DEFAULT_REF="a0afb5a56acd106b23a2b2385b8469834dc268c0" +DEFAULT_REF="80f301ff9a3b02c2c1e7be2dd1a567752f7b51b6" TARGET_REF="${VERSION:-$DEFAULT_REF}" echo "Checking out $TARGET_REF..." git checkout -f "$TARGET_REF" echo "Building from: $(git log --oneline -1)" # 3. Ensure shared library target exists -if ! grep -q "linkshared" c/BUILD; then +if ! grep -q '"libLiteRtLm.dylib"' c/BUILD; then echo "Adding shared library target to c/BUILD..." cat >> c/BUILD << 'BUILDEOF' diff --git a/packages/flutter_gemma_litertlm/native/litert_lm/build_macos.sh b/packages/flutter_gemma_litertlm/native/litert_lm/build_macos.sh index 729ff83a..b1acc00c 100755 --- a/packages/flutter_gemma_litertlm/native/litert_lm/build_macos.sh +++ b/packages/flutter_gemma_litertlm/native/litert_lm/build_macos.sh @@ -22,7 +22,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PREBUILT_DIR="$SCRIPT_DIR/prebuilt/macos_arm64" LITERT_LM_DIR="/tmp/LiteRT-LM" -DEFAULT_REF="a0afb5a56acd106b23a2b2385b8469834dc268c0" +DEFAULT_REF="80f301ff9a3b02c2c1e7be2dd1a567752f7b51b6" VERSION="${1:-}" echo "=== Building libLiteRtLm.dylib for macOS arm64 ===" @@ -48,7 +48,7 @@ git checkout -f "$TARGET_REF" echo "Building from: $(git log --oneline -1)" # 3. Ensure cc_binary(linkshared=True) target exists in c/BUILD -if ! grep -q "linkshared" c/BUILD; then +if ! grep -q '"libLiteRtLm.dylib"' c/BUILD; then echo "Adding shared library target to c/BUILD..." cat >> c/BUILD << 'BUILDEOF' diff --git a/packages/flutter_gemma_litertlm/native/litert_lm/patch_c_api.sh b/packages/flutter_gemma_litertlm/native/litert_lm/patch_c_api.sh index 31a45fe1..c211b2d0 100755 --- a/packages/flutter_gemma_litertlm/native/litert_lm/patch_c_api.sh +++ b/packages/flutter_gemma_litertlm/native/litert_lm/patch_c_api.sh @@ -1,8 +1,19 @@ #!/bin/bash # Patch LiteRT-LM C API source to add: -# 1. cc_binary(linkshared=True) target for shared library -# 2. litert_lm_engine_settings_set_max_num_images function -# 3. set_cache_dir propagation to vision/audio executors +# 1. cc_binary(linkshared=True) target for shared library (Linux dynamic-list +# + Windows .def whitelist for the symbols our Dart FFI bindings call) +# 2. set_use_hw_masking_for_npu setter (Intel LunarLake/PantherLake NPU) +# 3. GPU smooth-UI knobs — gpu_context_low_priority + kernel_batch_size (#364) +# 4. gpu_registry.cc dlopen rewrite for App-Store-safe framework paths (Apple) +# 5. minizip/zlib source mirrored off the flaky zlib.net (CI reliability) +# +# v0.14.0 migration (Phase 0): upstream 80f301f natively added a per-session +# sampler C API, set_max_num_images, set_litert_dispatch_lib_dir, set_cache_dir +# propagation, and the no-arg create()+setter conversation-config pattern — +# this obsoleted the former sections 2/3/4/5/6/7/8/9 of this script (they +# duplicated/monkey-patched now-native functionality, and two of them targeted +# session_basic.cc internals upstream deleted). Removed at the bump; kept +# section numbers below are the surviving originals, not renumbered. # # Usage: patch_c_api.sh # Example: patch_c_api.sh /tmp/LiteRT-LM @@ -24,7 +35,7 @@ echo "Patching LiteRT-LM C API in $DIR..." # libLiteRtLm.so references libLiteRt.so dynamically at runtime instead # of statically linking the LiteRt C API (which would create two copies # of TFLite in the process alongside the prebuilt accelerator). -if ! grep -q "linkshared" "$DIR/c/BUILD"; then +if ! grep -q '"libLiteRtLm.dylib"' "$DIR/c/BUILD"; then # Dynamic-list: make these symbols visible in the dynamic export table. cat > "$DIR/c/dynamic_list.lds" << 'LDSEOF' { @@ -50,6 +61,12 @@ EXPORTS litert_lm_conversation_cancel_process litert_lm_conversation_config_create litert_lm_conversation_config_delete + litert_lm_conversation_config_set_enable_constrained_decoding + litert_lm_conversation_config_set_extra_context + litert_lm_conversation_config_set_messages + litert_lm_conversation_config_set_session_config + litert_lm_conversation_config_set_system_message + litert_lm_conversation_config_set_tools litert_lm_conversation_create litert_lm_conversation_delete litert_lm_conversation_get_benchmark_info @@ -79,8 +96,17 @@ EXPORTS litert_lm_responses_delete litert_lm_responses_get_num_candidates litert_lm_responses_get_response_text_at + litert_lm_sampler_params_create + litert_lm_sampler_params_delete + litert_lm_sampler_params_set_seed + litert_lm_sampler_params_set_temperature + litert_lm_sampler_params_set_top_k + litert_lm_sampler_params_set_top_p litert_lm_session_config_create litert_lm_session_config_delete + litert_lm_session_config_set_apply_prompt_template + litert_lm_session_config_set_audio_lora_path + litert_lm_session_config_set_lora_path litert_lm_session_config_set_max_output_tokens litert_lm_session_config_set_sampler_params litert_lm_session_delete @@ -122,121 +148,6 @@ else echo " SKIP: c/BUILD already has shared lib target" fi -# ── 2. Add set_max_num_images to c/engine.h ── -if ! grep -q "set_max_num_images" "$DIR/c/engine.h"; then - # Insert before "Creates a LiteRT LM Engine from the given settings" - sed -i.bak '/Creates a LiteRT LM Engine from the given settings/i\ -// Sets the maximum number of images for multimodal vision support.\ -// Required for models with vision capabilities (e.g. Gemma 4, Gemma 3n).\ -LITERT_LM_C_API_EXPORT\ -void litert_lm_engine_settings_set_max_num_images(\ - LiteRtLmEngineSettings* settings, int max_num_images);\ -' "$DIR/c/engine.h" - rm -f "$DIR/c/engine.h.bak" - echo " OK: Added set_max_num_images to c/engine.h" -else - echo " SKIP: c/engine.h already has set_max_num_images" -fi - -# ── 3. Add set_max_num_images impl + patch set_cache_dir in c/engine.cc ── -if ! grep -q "set_max_num_images" "$DIR/c/engine.cc"; then - # Add set_max_num_images before set_activation_data_type - sed -i.bak '/void litert_lm_engine_settings_set_activation_data_type/i\ -void litert_lm_engine_settings_set_max_num_images(\ - LiteRtLmEngineSettings* settings, int max_num_images) {\ - if (settings \&\& settings->settings) {\ - settings->settings->GetMutableMainExecutorSettings().SetMaxNumImages(\ - max_num_images);\ - }\ -}\ -' "$DIR/c/engine.cc" - rm -f "$DIR/c/engine.cc.bak" - echo " OK: Added set_max_num_images impl to c/engine.cc" -else - echo " SKIP: c/engine.cc already has set_max_num_images" -fi - -# Patch set_cache_dir to also set on vision/audio executors -if ! grep -q "GetMutableVisionExecutorSettings.*SetCacheDir" "$DIR/c/engine.cc"; then - python3 -c " -import sys -with open('$DIR/c/engine.cc', 'r') as f: - content = f.read() - -old = '''void litert_lm_engine_settings_set_cache_dir(LiteRtLmEngineSettings* settings, - const char* cache_dir) { - if (settings && settings->settings) { - settings->settings->GetMutableMainExecutorSettings().SetCacheDir(cache_dir); - } -}''' - -new = '''void litert_lm_engine_settings_set_cache_dir(LiteRtLmEngineSettings* settings, - const char* cache_dir) { - if (settings && settings->settings) { - settings->settings->GetMutableMainExecutorSettings().SetCacheDir(cache_dir); - if (settings->settings->GetVisionExecutorSettings().has_value()) { - settings->settings->GetMutableVisionExecutorSettings()->SetCacheDir(cache_dir); - } - if (settings->settings->GetAudioExecutorSettings().has_value()) { - settings->settings->GetMutableAudioExecutorSettings()->SetCacheDir(cache_dir); - } - } -}''' - -if old in content: - content = content.replace(old, new) - with open('$DIR/c/engine.cc', 'w') as f: - f.write(content) - print(' OK: Patched set_cache_dir to propagate to vision/audio executors') -else: - print(' SKIP: set_cache_dir already patched or different format') -" -else - echo " SKIP: set_cache_dir already propagates to vision/audio" -fi - -# ── 4. Add set_litert_dispatch_lib_dir for accelerator plugin discovery ── -# On iOS the gpu_registry dlopens accelerator dylibs by relative basename, -# which dyld 4 cannot resolve in a sandboxed app. Expose SetLitertDispatchLibDir -# so callers (Dart FFI) can pass the absolute Frameworks directory. -if ! grep -q "set_litert_dispatch_lib_dir" "$DIR/c/engine.h"; then - sed -i.bak '/Creates a LiteRT LM Engine from the given settings/i\ -// Sets the directory where LiteRT dispatch libraries (e.g. accelerator\ -// plugins like libLiteRtMetalAccelerator.dylib) are located. On iOS this\ -// must be set to the absolute path of the app bundle Frameworks directory\ -// because dyld cannot resolve plugin libraries by basename in app sandboxes.\ -LITERT_LM_C_API_EXPORT\ -void litert_lm_engine_settings_set_litert_dispatch_lib_dir(\ - LiteRtLmEngineSettings* settings, const char* lib_dir);\ -' "$DIR/c/engine.h" - rm -f "$DIR/c/engine.h.bak" - echo " OK: Added set_litert_dispatch_lib_dir to c/engine.h" -else - echo " SKIP: c/engine.h already has set_litert_dispatch_lib_dir" -fi - -if ! grep -q "set_litert_dispatch_lib_dir" "$DIR/c/engine.cc"; then - sed -i.bak '/void litert_lm_engine_settings_set_activation_data_type/i\ -void litert_lm_engine_settings_set_litert_dispatch_lib_dir(\ - LiteRtLmEngineSettings* settings, const char* lib_dir) {\ - if (settings \&\& settings->settings \&\& lib_dir) {\ - settings->settings->GetMutableMainExecutorSettings().SetLitertDispatchLibDir(\ - lib_dir);\ - if (settings->settings->GetVisionExecutorSettings().has_value()) {\ - settings->settings->GetMutableVisionExecutorSettings()->SetLitertDispatchLibDir(lib_dir);\ - }\ - if (settings->settings->GetAudioExecutorSettings().has_value()) {\ - settings->settings->GetMutableAudioExecutorSettings()->SetLitertDispatchLibDir(lib_dir);\ - }\ - }\ -}\ -' "$DIR/c/engine.cc" - rm -f "$DIR/c/engine.cc.bak" - echo " OK: Added set_litert_dispatch_lib_dir impl to c/engine.cc" -else - echo " SKIP: c/engine.cc already has set_litert_dispatch_lib_dir" -fi - # ── 4b. Add set_use_hw_masking_for_npu for Intel LunarLake/PantherLake ── # Default NpuConfig.use_hw_masking_for_npu=true makes LiteRT setup HW mask # update path (MaskUpdateMethod::kWH) which Intel preview NPU silicon @@ -342,336 +253,6 @@ else echo " SKIP: c/engine.cc already has set_gpu_context_low_priority" fi -# ── 5. Patch litert_lm_conversation_config_create to 6-arg signature ── -# Upstream 5e0d86b ships the new no-args + setter pattern: -# create() -> set_session_config -> set_system_message -> set_tools -> ... -# but our Dart bindings.dart and patched engine.h declare a 6-arg monolithic -# overload because it pre-dates the upstream split. The mismatch was silent: -# our 6-arg call ABI-compatibly resolved to the 0-arg create() (extra args -# discarded), so session_config + sampler params were always lost — every -# inference ran with model defaults. -# -# Fix: rewrite upstream's no-args create() to be the 6-arg version we -# need. Internally apply each non-null arg through the existing setter -# functions. This keeps bindings.dart unchanged. -if ! grep -q "// PATCH: 6-arg overload" "$DIR/c/engine.cc"; then - python3 -c " -with open('$DIR/c/engine.cc', 'r') as f: - content = f.read() - -old = '''LiteRtLmConversationConfig* litert_lm_conversation_config_create() { - return new LiteRtLmConversationConfig; -}''' - -new = '''LiteRtLmConversationConfig* litert_lm_conversation_config_create( - LiteRtLmEngine* engine, const LiteRtLmSessionConfig* session_config, - const char* system_message_json, const char* tools_json, - const char* messages_json, bool enable_constrained_decoding) { - // PATCH: 6-arg overload of upstream's no-args create() — keeps our - // Dart bindings.dart in sync with the existing C++ setter functions. - // engine pointer is currently unused (kept in signature for symmetry - // with upstream proposed monolithic API). - (void)engine; - auto* config = new LiteRtLmConversationConfig; - if (session_config && session_config->config) { - config->session_config = *session_config->config; - } - if (system_message_json) { - config->system_message_json = system_message_json; - } - if (tools_json) { - config->tools_json = tools_json; - } - if (messages_json) { - config->messages_json = messages_json; - } - config->enable_constrained_decoding = enable_constrained_decoding; - return config; -}''' - -if old in content: - content = content.replace(old, new) - with open('$DIR/c/engine.cc', 'w') as f: - f.write(content) - print(' OK: Patched litert_lm_conversation_config_create to 6-arg overload') -else: - print(' SKIP: config_create already patched or different format') -" -else - echo " SKIP: c/engine.cc already has 6-arg config_create patch" -fi - -# Also patch the upstream header to expose the 6-arg signature so -# downstream Bazel users (and our generated bindings) match the impl. -if grep -q "LiteRtLmConversationConfig\* litert_lm_conversation_config_create();" "$DIR/c/engine.h"; then - python3 -c " -with open('$DIR/c/engine.h', 'r') as f: - content = f.read() - -old = 'LiteRtLmConversationConfig* litert_lm_conversation_config_create();' -new = '''LiteRtLmConversationConfig* litert_lm_conversation_config_create( - LiteRtLmEngine* engine, const LiteRtLmSessionConfig* session_config, - const char* system_message_json, const char* tools_json, - const char* messages_json, bool enable_constrained_decoding);''' -if old in content: - content = content.replace(old, new) - with open('$DIR/c/engine.h', 'w') as f: - f.write(content) - print(' OK: Patched c/engine.h declaration to 6-arg signature') -" -fi - -# ── 6. Patch LlmExecutor base to add SetPendingSamplerParams virtual ── -# Upstream's LlmLiteRtCompiledModelExecutorBase::InitializeSampler() at -# llm_litert_compiled_model_executor.cc:1271-1276 hardcodes sampler_params -# (TOP_P, k=1, p=0, temperature=1, seed=0) ignoring SessionConfig. We add -# a new virtual on the LlmExecutor base so SessionBasic can push session- -# level sampler params into the executor BEFORE InitializeSampler runs. -# -# Defaulted to UnimplementedError so executor implementations that don't -# override (e.g. NPU executor) keep current upstream behavior — they just -# won't honor seed via this path. -EXECUTOR_BASE_HEADER="$DIR/runtime/executor/llm_executor_base.h" -if [ -f "$EXECUTOR_BASE_HEADER" ] && ! grep -q "// PATCH: SetPendingSamplerParams" "$EXECUTOR_BASE_HEADER"; then - python3 -c " -with open('$EXECUTOR_BASE_HEADER', 'r') as f: - content = f.read() - -# Anchor: insert the new virtual right before UpdateExecutorSettings. -# UpdateExecutorSettings is the conventional 'change runtime config' verb, -# so SetPendingSamplerParams lives next to it. -import re -m = re.search(r'( virtual absl::Status UpdateExecutorSettings\\b)', content) -if not m: - print(' WARN: UpdateExecutorSettings not found in llm_executor_base.h; skipping section 6') -else: - insertion = ''' // PATCH: SetPendingSamplerParams (flutter_gemma). - // Inject session-level sampler params so InitializeSampler() honors the - // user's seed/temperature/topK/topP instead of upstream's hardcoded - // defaults at llm_litert_compiled_model_executor.cc:1271-1276. - // Defaulted to UnimplementedError so executors that don't override keep - // current behavior. - virtual absl::Status SetPendingSamplerParams( - const proto::SamplerParameters& sampler_params) { - return absl::UnimplementedError( - \"SetPendingSamplerParams not implemented for this executor.\"); - } - -''' - content = content[:m.start()] + insertion + content[m.start():] - # Ensure the proto include is present (it's a transitive dep via the - # sampler classes, but be explicit). - if 'runtime/proto/sampler_params.pb.h' not in content: - # Insert near other runtime includes. - m2 = re.search(r'(#include \"runtime/[^\"]+\"\\n)', content) - if m2: - content = content[:m2.end()] + '#include \"runtime/proto/sampler_params.pb.h\"\\n' + content[m2.end():] - with open('$EXECUTOR_BASE_HEADER', 'w') as f: - f.write(content) - print(' OK: Added SetPendingSamplerParams virtual to llm_executor_base.h') -" -else - if [ -f "$EXECUTOR_BASE_HEADER" ]; then - echo " SKIP: llm_executor_base.h already has SetPendingSamplerParams" - else - echo " WARN: $EXECUTOR_BASE_HEADER not found" - fi -fi - -# ── 7. Patch LlmLiteRtCompiledModelExecutorBase to add override + member ── -LITERT_EXECUTOR_HEADER="$DIR/runtime/executor/llm_litert_compiled_model_executor.h" -if [ -f "$LITERT_EXECUTOR_HEADER" ] && ! grep -q "// PATCH: pending_sampler_params_" "$LITERT_EXECUTOR_HEADER"; then - python3 -c " -with open('$LITERT_EXECUTOR_HEADER', 'r') as f: - content = f.read() - -import re - -# 7a: add override declaration after the existing InitializeSampler decl. -m = re.search( - r'( absl::Status InitializeSampler\\(\\s*std::optional[^)]*\\)\\s*;\\s*\\n)', - content) -if not m: - print(' WARN: InitializeSampler decl not found; skipping section 7a') -else: - override_decl = ''' - // PATCH: SetPendingSamplerParams (flutter_gemma). - // See llm_executor_base.h. Stash and forward to UpdateConfig if the - // sampler already exists (subsequent createSession on the same engine). - absl::Status SetPendingSamplerParams( - const proto::SamplerParameters& sampler_params) override; - -''' - content = content[:m.end()] + override_decl + content[m.end():] - -# 7b: add member field after the existing sampler_ declaration. -m2 = re.search(r'( std::unique_ptr sampler_;\\n)', content) -if not m2: - print(' WARN: sampler_ field not found; skipping section 7b') -else: - field_decl = ''' - // PATCH: pending_sampler_params_ (flutter_gemma). - // User-supplied sampler params from SessionConfig; consumed by - // InitializeSampler() if set, else fall back to upstream hardcoded - // defaults. nullopt preserves behavior for callers that don't push - // session params. - std::optional pending_sampler_params_; -''' - content = content[:m2.end()] + field_decl + content[m2.end():] - -with open('$LITERT_EXECUTOR_HEADER', 'w') as f: - f.write(content) -print(' OK: Added SetPendingSamplerParams + pending_sampler_params_ to litert_compiled_model_executor.h') -" -else - if [ -f "$LITERT_EXECUTOR_HEADER" ]; then - echo " SKIP: litert_compiled_model_executor.h already patched" - else - echo " WARN: $LITERT_EXECUTOR_HEADER not found" - fi -fi - -# ── 8. Patch InitializeSampler + add SetPendingSamplerParams definition ── -LITERT_EXECUTOR_CC="$DIR/runtime/executor/llm_litert_compiled_model_executor.cc" -if [ -f "$LITERT_EXECUTOR_CC" ] && ! grep -q "// PATCH: pending_sampler_params_ in InitializeSampler" "$LITERT_EXECUTOR_CC"; then - python3 -c " -with open('$LITERT_EXECUTOR_CC', 'r') as f: - content = f.read() - -# 8a: replace hardcoded SamplerParameters block in InitializeSampler. -old = ''' proto::SamplerParameters sampler_params; - sampler_params.set_type(proto::SamplerParameters::TOP_P); - sampler_params.set_k(1); - sampler_params.set_p(0.0f); - sampler_params.set_temperature(1.0f); - sampler_params.set_seed(0); - ASSIGN_OR_RETURN( - sampler_, - CreateSampler(sampler_backend, output_heads, std::move(sampler_params), - env_.Get(), /*sequence_size=*/1, vocab_size, data_type));''' - -new = ''' // PATCH: pending_sampler_params_ in InitializeSampler (flutter_gemma). - // Consume session-supplied sampler params if present; otherwise - // preserve upstream's hardcoded defaults so non-flutter_gemma callers - // see no behavior change. - proto::SamplerParameters sampler_params; - if (pending_sampler_params_.has_value()) { - sampler_params = *pending_sampler_params_; - if (sampler_params.type() == proto::SamplerParameters::TYPE_UNSPECIFIED) { - sampler_params.set_type(proto::SamplerParameters::TOP_P); - } - if (sampler_params.k() <= 0) sampler_params.set_k(1); - // p == 0.0 is a legitimate \"no top-p\" choice; don\\'t backfill. - if (sampler_params.temperature() <= 0.0f) { - sampler_params.set_temperature(1.0f); - } - // seed == 0 is a legitimate user choice; don\\'t backfill. - } else { - sampler_params.set_type(proto::SamplerParameters::TOP_P); - sampler_params.set_k(1); - sampler_params.set_p(0.0f); - sampler_params.set_temperature(1.0f); - sampler_params.set_seed(0); - } - ASSIGN_OR_RETURN( - sampler_, - CreateSampler(sampler_backend, output_heads, std::move(sampler_params), - env_.Get(), /*sequence_size=*/1, vocab_size, data_type));''' - -if old in content: - content = content.replace(old, new) - print(' OK: Patched InitializeSampler to consume pending_sampler_params_') -else: - print(' WARN: hardcoded sampler_params block not found in cc; skipping section 8a') - -# 8b: add SetPendingSamplerParams definition after UpdateExecutorSettings. -import re -m = re.search( - r'(absl::Status LlmLiteRtCompiledModelExecutorBase::UpdateExecutorSettings\\([^)]*\\)\\s*\\{[^}]*\\}\\n)', - content) -if not m: - print(' WARN: UpdateExecutorSettings impl not found; skipping section 8b') -else: - setter_def = ''' -// PATCH: SetPendingSamplerParams (flutter_gemma). -absl::Status LlmLiteRtCompiledModelExecutorBase::SetPendingSamplerParams( - const proto::SamplerParameters& sampler_params) { - pending_sampler_params_ = sampler_params; - // Always drop the cached sampler so the next InitializeSampler() call - // recreates it with our pending_sampler_params_. We can't rely on - // Sampler::UpdateConfig() because: - // - top_p_cpu_sampler.cc:168 ignores sampler_params.seed() — only k/p/ - // temperature are mutated; the std::default_random_engine is left - // as-is, so the same input seed across two calls produces different - // outputs (RNG state accumulates). - // - GPU C-API sampler libs (Metal #1990) ship without the UpdateConfig - // export at all on some platforms. - // Recreate-on-set is the only reliable way to honor a fresh seed. - sampler_.reset(); - return absl::OkStatus(); -} -''' - content = content[:m.end()] + setter_def + content[m.end():] - print(' OK: Added SetPendingSamplerParams definition to litert_compiled_model_executor.cc') - -with open('$LITERT_EXECUTOR_CC', 'w') as f: - f.write(content) -" -else - if [ -f "$LITERT_EXECUTOR_CC" ]; then - echo " SKIP: litert_compiled_model_executor.cc already patched" - else - echo " WARN: $LITERT_EXECUTOR_CC not found" - fi -fi - -# ── 9. Patch SessionBasic::Create to push sampler params on GPU/NPU path ── -SESSION_BASIC_CC="$DIR/runtime/core/session_basic.cc" -if [ -f "$SESSION_BASIC_CC" ] && ! grep -q "// PATCH: push sampler params on GPU/NPU" "$SESSION_BASIC_CC"; then - python3 -c " -with open('$SESSION_BASIC_CC', 'r') as f: - content = f.read() - -# Find the existing GPU/NPU else-if branch and replace it. -old = ''' } else if (sampler_backend != Backend::GPU && - sampler_backend != Backend::NPU) { - return absl::InvalidArgumentError( - absl::StrCat(\"Unsupported sampler backend: \", sampler_backend)); - }''' - -new = ''' } else if (sampler_backend == Backend::GPU || - sampler_backend == Backend::NPU) { - // PATCH: push sampler params on GPU/NPU (flutter_gemma). - // Push session-level sampler params into the executor so its - // InitializeSampler() consumes user's seed/temperature/topK/topP - // instead of hardcoded defaults. Best-effort: NPU executor returns - // Unimplemented and we ignore it (preserves upstream behavior). - auto status = executor->SetPendingSamplerParams( - session_config.GetSamplerParams()); - if (!status.ok() && status.code() != absl::StatusCode::kUnimplemented) { - return status; - } - } else { - return absl::InvalidArgumentError( - absl::StrCat(\"Unsupported sampler backend: \", sampler_backend)); - }''' - -if old in content: - content = content.replace(old, new) - with open('$SESSION_BASIC_CC', 'w') as f: - f.write(content) - print(' OK: Patched SessionBasic::Create to push sampler params on GPU/NPU') -else: - print(' WARN: existing GPU/NPU else-if branch not found in session_basic.cc; skipping section 9') -" -else - if [ -f "$SESSION_BASIC_CC" ]; then - echo " SKIP: session_basic.cc already patched" - else - echo " WARN: $SESSION_BASIC_CC not found" - fi -fi - # ── 10. Patch sampler_factory.cc + WORKSPACE for App-Store-safe dlopen on Apple ── # # `gpu_registry.cc` (LiteRT) and `sampler_factory.cc` (LiteRT-LM) hardcode the From 9c89ad45b5e0dc6c329d035c6c6fc32ddff0ad09 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Mon, 20 Jul 2026 00:22:15 +0400 Subject: [PATCH 2/8] feat(litertlm): migrate FFI to LiteRT-LM v0.14.0 session-config + opaque sampler API (Phase 2) --- .../lib/src/ffi/litert_lm_bindings.dart | 171 ++++++++++++------ .../lib/src/ffi/litert_lm_client.dart | 61 +++++-- 2 files changed, 160 insertions(+), 72 deletions(-) diff --git a/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_bindings.dart b/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_bindings.dart index 3240ca5c..60c66bfc 100644 --- a/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_bindings.dart +++ b/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_bindings.dart @@ -117,49 +117,23 @@ class LiteRtLmBindings { _litert_lm_session_config_deletePtr .asFunction)>(); - ffi.Pointer litert_lm_conversation_config_create( - ffi.Pointer engine, - ffi.Pointer session_config, - ffi.Pointer system_message_json, - ffi.Pointer tools_json, - ffi.Pointer messages_json, - bool enable_constrained_decoding, - ) { - return _litert_lm_conversation_config_create( - engine, - session_config, - system_message_json, - tools_json, - messages_json, - enable_constrained_decoding, - ); + // No-arg as of LiteRT-LM v0.14.0 (was a 6-arg overload taking engine + + // session_config + system/tools/messages JSON + enable_constrained_decoding + // directly, added by our now-deleted native/litert_lm/patch_c_api.sh + // patch). Build the config via this create() plus the + // litert_lm_conversation_config_set_* setter chain below. + ffi.Pointer + litert_lm_conversation_config_create() { + return _litert_lm_conversation_config_create(); } late final _litert_lm_conversation_config_createPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Bool, - ) - > + ffi.NativeFunction Function()> >('litert_lm_conversation_config_create'); late final _litert_lm_conversation_config_create = _litert_lm_conversation_config_createPtr - .asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - bool, - ) - >(); + .asFunction Function()>(); void litert_lm_conversation_config_set_session_config( ffi.Pointer config, @@ -1946,6 +1920,108 @@ class LiteRtLmBindings { ffi.Pointer, ) >(); + + ffi.Pointer litert_lm_sampler_params_create(int type) { + return _litert_lm_sampler_params_create(type); + } + + late final _litert_lm_sampler_params_createPtr = + _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Int32) + > + >('litert_lm_sampler_params_create'); + late final _litert_lm_sampler_params_create = + _litert_lm_sampler_params_createPtr + .asFunction Function(int)>(); + + void litert_lm_sampler_params_delete( + ffi.Pointer params, + ) { + return _litert_lm_sampler_params_delete(params); + } + + late final _litert_lm_sampler_params_deletePtr = + _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer) + > + >('litert_lm_sampler_params_delete'); + late final _litert_lm_sampler_params_delete = + _litert_lm_sampler_params_deletePtr + .asFunction)>(); + + void litert_lm_sampler_params_set_top_k( + ffi.Pointer params, + int top_k, + ) { + return _litert_lm_sampler_params_set_top_k(params, top_k); + } + + late final _litert_lm_sampler_params_set_top_kPtr = + _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Int32) + > + >('litert_lm_sampler_params_set_top_k'); + late final _litert_lm_sampler_params_set_top_k = + _litert_lm_sampler_params_set_top_kPtr + .asFunction, int)>(); + + void litert_lm_sampler_params_set_top_p( + ffi.Pointer params, + double top_p, + ) { + return _litert_lm_sampler_params_set_top_p(params, top_p); + } + + late final _litert_lm_sampler_params_set_top_pPtr = + _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Float) + > + >('litert_lm_sampler_params_set_top_p'); + late final _litert_lm_sampler_params_set_top_p = + _litert_lm_sampler_params_set_top_pPtr + .asFunction< + void Function(ffi.Pointer, double) + >(); + + void litert_lm_sampler_params_set_temperature( + ffi.Pointer params, + double temperature, + ) { + return _litert_lm_sampler_params_set_temperature(params, temperature); + } + + late final _litert_lm_sampler_params_set_temperaturePtr = + _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Float) + > + >('litert_lm_sampler_params_set_temperature'); + late final _litert_lm_sampler_params_set_temperature = + _litert_lm_sampler_params_set_temperaturePtr + .asFunction< + void Function(ffi.Pointer, double) + >(); + + void litert_lm_sampler_params_set_seed( + ffi.Pointer params, + int seed, + ) { + return _litert_lm_sampler_params_set_seed(params, seed); + } + + late final _litert_lm_sampler_params_set_seedPtr = + _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Int32) + > + >('litert_lm_sampler_params_set_seed'); + late final _litert_lm_sampler_params_set_seed = + _litert_lm_sampler_params_set_seedPtr + .asFunction, int)>(); } final class LiteRtLmEngine extends ffi.Opaque {} @@ -2010,24 +2086,11 @@ enum LiteRtLmSamplerType { }; } -final class LiteRtLmSamplerParams extends ffi.Struct { - @ffi.UnsignedInt() - external int typeAsInt; - - LiteRtLmSamplerType get type => LiteRtLmSamplerType.fromValue(typeAsInt); - - @ffi.Int32() - external int top_k; - - @ffi.Float() - external double top_p; - - @ffi.Float() - external double temperature; - - @ffi.Int32() - external int seed; -} +// Opaque as of LiteRT-LM v0.14.0 (was a transparent struct with +// typeAsInt/top_k/top_p/temperature/seed fields). Build via +// litert_lm_sampler_params_create + the litert_lm_sampler_params_set_* +// setters; fields can no longer be poked directly. +final class LiteRtLmSamplerParams extends ffi.Opaque {} enum LiteRtLmInputDataType { kLiteRtLmInputDataTypeText(0), diff --git a/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_client.dart b/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_client.dart index 24c13803..eaacbea3 100644 --- a/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_client.dart +++ b/packages/flutter_gemma_litertlm/lib/src/ffi/litert_lm_client.dart @@ -895,10 +895,12 @@ class LiteRtLmFfiClient { // falls back to its baked-in defaults (typically greedy), making // every call ignore stochastic decoding requests. // - // This requires a patched libLiteRtLm.{so,dylib,dll} where - // litert_lm_conversation_config_create accepts the 6-arg overload - // and applies session_config via the upstream setter chain. See - // native/litert_lm/patch_c_api.sh ("PATCH: 6-arg overload"). + // LiteRT-LM v0.14.0: litert_lm_conversation_config_create() takes no + // arguments — session_config, system message, tools, and messages are + // attached afterwards via the litert_lm_conversation_config_set_* + // setter chain built below. This is now real upstream API; the old + // 6-arg overload from the now-deleted + // native/litert_lm/patch_c_api.sh ("PATCH: 6-arg overload") is gone. final sessionConfig = b.litert_lm_session_config_create(); // NPU executor on LiteRT-LM only supports internal greedy sampling — any @@ -906,22 +908,25 @@ class LiteRtLmFfiClient { // upstream. Skip the setter chain in that case; CPU/GPU paths are // unaffected. if (_backend != 'npu') { - final samplerParams = calloc(); // Upstream LiteRT-LM (commit 5e0d86b) only implements TopP sampling at // engine level — sampler type 1 (TopK) and 3 (Greedy) are rejected with // "UNIMPLEMENTED: Sampler type: N not implemented yet." Use TopP (=2) // unconditionally and pass top_k as a hint; native respects both fields // even though it's gated by the type tag. - samplerParams.ref.typeAsInt = 2; // always TopP - samplerParams.ref.top_k = topK; - samplerParams.ref.top_p = topP ?? 0.95; - samplerParams.ref.temperature = temperature; - samplerParams.ref.seed = seed; + // + // v0.14.0: LiteRtLmSamplerParams is opaque — built via + // litert_lm_sampler_params_create + the _set_* setters instead of + // writing struct fields directly. + final samplerParams = b.litert_lm_sampler_params_create(2); // always TopP + b.litert_lm_sampler_params_set_top_k(samplerParams, topK); + b.litert_lm_sampler_params_set_top_p(samplerParams, topP ?? 0.95); + b.litert_lm_sampler_params_set_temperature(samplerParams, temperature); + b.litert_lm_sampler_params_set_seed(samplerParams, seed); b.litert_lm_session_config_set_sampler_params( sessionConfig, samplerParams, ); - calloc.free(samplerParams); + b.litert_lm_sampler_params_delete(samplerParams); } else { gemmaLog( '[LiteRtLmFfi] NPU backend — sampler params ' @@ -954,15 +959,35 @@ class LiteRtLmFfiClient { final toolsPtr = toolsJson?.toNativeUtf8(); final messagesPtr = messagesJson?.toNativeUtf8(); + // v0.14.0: no-arg create + setter chain (engine is no longer an + // argument — session_config carries everything the native side needs). final Pointer convConfig = b - .litert_lm_conversation_config_create( - _engine!, - sessionConfig, - systemPtr?.cast() ?? nullptr, - toolsPtr?.cast() ?? nullptr, - messagesPtr?.cast() ?? nullptr, - toolsJson != null, + .litert_lm_conversation_config_create(); + if (convConfig != nullptr) { + b.litert_lm_conversation_config_set_session_config( + convConfig, + sessionConfig, + ); + if (systemPtr != null) { + b.litert_lm_conversation_config_set_system_message( + convConfig, + systemPtr.cast(), + ); + } + if (toolsPtr != null) { + b.litert_lm_conversation_config_set_tools(convConfig, toolsPtr.cast()); + b.litert_lm_conversation_config_set_enable_constrained_decoding( + convConfig, + true, + ); + } + if (messagesPtr != null) { + b.litert_lm_conversation_config_set_messages( + convConfig, + messagesPtr.cast(), ); + } + } b.litert_lm_session_config_delete(sessionConfig); if (systemPtr != null) calloc.free(systemPtr); From 6a03da1a7be271ff4ad30f744646f161b05ed700 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Mon, 20 Jul 2026 10:27:41 +0400 Subject: [PATCH 3/8] fix(embeddings): migrate LiteRtCreateModelFromFile to the 3-arg LiteRT v0.14.0 signature (adds LiteRtEnvironment) --- .../lib/src/litert/litert_bindings.dart | 7 +++++-- .../lib/src/litert/litert_embedding_core.dart | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/flutter_gemma_embeddings/lib/src/litert/litert_bindings.dart b/packages/flutter_gemma_embeddings/lib/src/litert/litert_bindings.dart index 2eb84e21..423e78f1 100644 --- a/packages/flutter_gemma_embeddings/lib/src/litert/litert_bindings.dart +++ b/packages/flutter_gemma_embeddings/lib/src/litert/litert_bindings.dart @@ -318,10 +318,13 @@ class LiteRtBindings { // Model. + // LiteRT (622f1f3c / LiteRT-LM v0.14.0) added a leading LiteRtEnvironment + // parameter to LiteRtCreateModelFromFile. Calling the old 2-arg form against + // the new lib read `filename` as `environment` → status=500 on every load. late final createModelFromFile = _lib .lookupFunction< - Int32 Function(Pointer, Pointer), - int Function(Pointer, Pointer) + Int32 Function(LiteRtEnvironment, Pointer, Pointer), + int Function(LiteRtEnvironment, Pointer, Pointer) >('LiteRtCreateModelFromFile'); late final destroyModel = _lib diff --git a/packages/flutter_gemma_embeddings/lib/src/litert/litert_embedding_core.dart b/packages/flutter_gemma_embeddings/lib/src/litert/litert_embedding_core.dart index 78256ff5..5685af3c 100644 --- a/packages/flutter_gemma_embeddings/lib/src/litert/litert_embedding_core.dart +++ b/packages/flutter_gemma_embeddings/lib/src/litert/litert_embedding_core.dart @@ -123,7 +123,7 @@ class EmbeddingCore { final modelPtr = calloc(); try { bindings - .createModelFromFile(pathC, modelPtr) + .createModelFromFile(environment, pathC, modelPtr) .check('LiteRtCreateModelFromFile($modelPath)'); } finally { calloc.free(pathC); From 822bcdf1273aa9e760820004c90a40217ee32f07 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Mon, 20 Jul 2026 16:15:11 +0400 Subject: [PATCH 4/8] fix(litertlm): bundle libwebgpu_dawn + $ORIGIN rpath for v0.14.0 Dawn split v0.14.0 split Dawn out of libLiteRtWebGpuAccelerator into a separate libwebgpu_dawn.{so,dll} (0.13.1 static-linked it), so the accelerator now hard-depends on it and desktop GPU fails to load (all backends) without it. - hooks: add 'webgpu_dawn' to companions (+ 'libwebgpu_dawn' to windowsExtraLibs for the PE import); bump native version to 0.14.0 - Linux CI: patchelf --set-rpath '$ORIGIN' so each lib finds its flat-layout siblings at dlopen time - Windows CI: bundle libwebgpu_dawn.dll in the Collect companion loop Validated on Linux (Tesla T4): litertlm smoke 24/24 pass. --- .github/workflows/build-litertlm-native-windows.yml | 2 +- .github/workflows/build-litertlm-native.yml | 12 +++++++++++- packages/flutter_gemma_embeddings/hook/build.dart | 4 +++- packages/flutter_gemma_litertlm/hook/build.dart | 4 +++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-litertlm-native-windows.yml b/.github/workflows/build-litertlm-native-windows.yml index 32610b42..6e846735 100644 --- a/.github/workflows/build-litertlm-native-windows.yml +++ b/.github/workflows/build-litertlm-native-windows.yml @@ -80,7 +80,7 @@ jobs: Copy-Item C:\LiteRT-LM\bazel-bin\c\libLiteRtLm.dylib artifacts\LiteRtLm.dll Copy-Item StreamProxy.dll artifacts\ $prebuilt = 'C:\LiteRT-LM\prebuilt\windows_x86_64' - foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator')) { + foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator','libwebgpu_dawn')) { $src = "$prebuilt\$base.dll" if (Test-Path $src) { Copy-Item $src "artifacts\$base.dll" diff --git a/.github/workflows/build-litertlm-native.yml b/.github/workflows/build-litertlm-native.yml index b7568fe5..83fff16d 100644 --- a/.github/workflows/build-litertlm-native.yml +++ b/.github/workflows/build-litertlm-native.yml @@ -63,6 +63,11 @@ jobs: cp /tmp/LiteRT-LM/bazel-bin/c/libLiteRtLm.dylib artifacts/libLiteRtLm.so cp /tmp/libStreamProxy.so artifacts/ cp /tmp/LiteRT-LM/prebuilt/linux_x86_64/*.so artifacts/ 2>/dev/null || true + # v0.14.0 Dawn split — libLiteRtWebGpuAccelerator.so now hard-links the + # separate libwebgpu_dawn.so. Set $ORIGIN rpath so each lib resolves its + # flat-layout siblings from its own directory at dlopen time. + sudo apt-get update -qq && sudo apt-get install -y -qq patchelf + for f in artifacts/*.so; do patchelf --set-rpath '$ORIGIN' "$f"; done ls -lh artifacts/ - uses: actions/upload-artifact@v4 @@ -111,6 +116,11 @@ jobs: cp /tmp/LiteRT-LM/bazel-bin/c/libLiteRtLm.dylib artifacts/libLiteRtLm.so cp /tmp/libStreamProxy.so artifacts/ cp /tmp/LiteRT-LM/prebuilt/linux_arm64/*.so artifacts/ 2>/dev/null || true + # v0.14.0 Dawn split — libLiteRtWebGpuAccelerator.so now hard-links the + # separate libwebgpu_dawn.so. Set $ORIGIN rpath so each lib resolves its + # flat-layout siblings from its own directory at dlopen time. + sudo apt-get update -qq && sudo apt-get install -y -qq patchelf + for f in artifacts/*.so; do patchelf --set-rpath '$ORIGIN' "$f"; done ls -lh artifacts/ - uses: actions/upload-artifact@v4 @@ -193,7 +203,7 @@ jobs: Copy-Item C:\LiteRT-LM\bazel-bin\c\libLiteRtLm.dylib artifacts\LiteRtLm.dll Copy-Item StreamProxy.dll artifacts\ $prebuilt = 'C:\LiteRT-LM\prebuilt\windows_x86_64' - foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator')) { + foreach ($base in @('libGemmaModelConstraintProvider','libLiteRt','libLiteRtTopKWebGpuSampler','libLiteRtWebGpuAccelerator','libwebgpu_dawn')) { $src = "$prebuilt\$base.dll" if (Test-Path $src) { # Keep original name for PE imports diff --git a/packages/flutter_gemma_embeddings/hook/build.dart b/packages/flutter_gemma_embeddings/hook/build.dart index 6ecd6018..93742b49 100644 --- a/packages/flutter_gemma_embeddings/hook/build.dart +++ b/packages/flutter_gemma_embeddings/hook/build.dart @@ -148,7 +148,7 @@ class _NativeBundle { /// (#245). Android: `-Wl,-z,max-page-size=16384` (Google Play 16KB). const _litertlmBundle = _NativeBundle( namespace: 'litertlm', - version: '0.13.1-b', + version: '0.14.0', releaseTagPrefix: 'native-v', archivePrefix: 'litertlm', mainLibName: 'LiteRtLm', @@ -198,6 +198,7 @@ const _litertlmBundle = _NativeBundle( 'LiteRtTopKOpenClSampler', // Android OpenCL GPU sampler — honors seed 'LiteRtTopKWebGpuSampler', // Linux/Windows GPU sampler 'LiteRt', // Linux/Windows core runtime + 'webgpu_dawn', // Linux/Windows Dawn WebGPU (split to a shared lib in v0.14.0) ], // On macOS, skip the upstream Apple companion dylibs from Native Assets // bundling (#247). The three dylibs Google ships in @@ -228,6 +229,7 @@ const _litertlmBundle = _NativeBundle( 'libLiteRt', 'libLiteRtTopKWebGpuSampler', 'libLiteRtWebGpuAccelerator', + 'libwebgpu_dawn', // lib-prefixed for the accelerator's PE import (v0.14.0) // DXC runtime 'dxil', 'dxcompiler', diff --git a/packages/flutter_gemma_litertlm/hook/build.dart b/packages/flutter_gemma_litertlm/hook/build.dart index 98ddfe6a..f9a503fe 100644 --- a/packages/flutter_gemma_litertlm/hook/build.dart +++ b/packages/flutter_gemma_litertlm/hook/build.dart @@ -148,7 +148,7 @@ class _NativeBundle { /// (#245). Android: `-Wl,-z,max-page-size=16384` (Google Play 16KB). const _litertlmBundle = _NativeBundle( namespace: 'litertlm', - version: '0.13.1-b', + version: '0.14.0', releaseTagPrefix: 'native-v', archivePrefix: 'litertlm', mainLibName: 'LiteRtLm', @@ -200,6 +200,7 @@ const _litertlmBundle = _NativeBundle( 'LiteRtTopKOpenClSampler', // Android OpenCL GPU sampler — honors seed 'LiteRtTopKWebGpuSampler', // Linux/Windows GPU sampler 'LiteRt', // Linux/Windows core runtime + 'webgpu_dawn', // Linux/Windows Dawn WebGPU (split to a shared lib in v0.14.0) ], // On macOS, skip the upstream Apple companion dylibs from Native Assets // bundling (#247). The three dylibs Google ships in @@ -230,6 +231,7 @@ const _litertlmBundle = _NativeBundle( 'libLiteRt', 'libLiteRtTopKWebGpuSampler', 'libLiteRtWebGpuAccelerator', + 'libwebgpu_dawn', // lib-prefixed for the accelerator's PE import (v0.14.0) // DXC runtime 'dxil', 'dxcompiler', From cead4ac056246abbaee737f0cdbdca33375b6a58 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Mon, 20 Jul 2026 16:37:25 +0400 Subject: [PATCH 5/8] ci(litertlm): chmod +w before patchelf (git-lfs prebuilt .so are read-only) --- .github/workflows/build-litertlm-native.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-litertlm-native.yml b/.github/workflows/build-litertlm-native.yml index 83fff16d..6c9496d1 100644 --- a/.github/workflows/build-litertlm-native.yml +++ b/.github/workflows/build-litertlm-native.yml @@ -67,6 +67,7 @@ jobs: # separate libwebgpu_dawn.so. Set $ORIGIN rpath so each lib resolves its # flat-layout siblings from its own directory at dlopen time. sudo apt-get update -qq && sudo apt-get install -y -qq patchelf + chmod u+w artifacts/*.so # git-lfs prebuilt .so are read-only; patchelf edits in place for f in artifacts/*.so; do patchelf --set-rpath '$ORIGIN' "$f"; done ls -lh artifacts/ @@ -120,6 +121,7 @@ jobs: # separate libwebgpu_dawn.so. Set $ORIGIN rpath so each lib resolves its # flat-layout siblings from its own directory at dlopen time. sudo apt-get update -qq && sudo apt-get install -y -qq patchelf + chmod u+w artifacts/*.so # git-lfs prebuilt .so are read-only; patchelf edits in place for f in artifacts/*.so; do patchelf --set-rpath '$ORIGIN' "$f"; done ls -lh artifacts/ From 150f886eb3f32959a853c6acb68442f95c3500b5 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Thu, 23 Jul 2026 00:40:10 +0400 Subject: [PATCH 6/8] =?UTF-8?q?feat(litertlm):=20LiteRT-LM=20v0.14.0=20mig?= =?UTF-8?q?ration=20=E2=80=94=20native=20per-session=20sampler=20+=20#214?= =?UTF-8?q?=20fix=20(litertlm=201.2.0,=20embeddings=201.0.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FFI on v0.14.0 opaque session-config/sampler API; native-v0.14.0 (7 platforms). - #214 GPU output-garbage fixed; macOS/Linux GPU + Windows CPU/NPU verified. - Web @litert-lm/core 0.12.1 -> 0.14.0 (text path). - Add community models (SmolLM3-3B, Phi-4-mini-reasoning, Qwen2-VL, SmolVLM2, LLaVA-OneVision) to example + README. - macOS: re-sign companion frameworks after install_name_tool (fixes CODESIGNING Invalid Page). - Known: Windows discrete GPU regressed upstream (LiteRT-LM #2957) — use CPU/NPU. --- CLAUDE.md | 2 +- packages/flutter_gemma/CHANGELOG.md | 4 + packages/flutter_gemma/README.md | 32 ++- .../community_models_smoke_test.dart | 184 ++++++++++++++++++ .../fastvlm_cpu_smoke_test.dart | 78 ++++++++ .../integration_test/fastvlm_smoke_test.dart | 75 +++++++ .../internvl35_smoke_test.dart | 75 +++++++ .../integration_test/llava_smoke_test.dart | 75 +++++++ .../integration_test/phi4_smoke_test.dart | 73 +++++++ .../integration_test/qwen25_smoke_test.dart | 61 ++++++ .../integration_test/qwen2vl_smoke_test.dart | 99 ++++++++++ .../integration_test/smollm3_smoke_test.dart | 83 ++++++++ .../integration_test/smolvlm2_smoke_test.dart | 75 +++++++ .../flutter_gemma/example/ios/Podfile.lock | 91 ++------- .../example/lib/models/model.dart | 112 ++++++++++- .../Flutter/GeneratedPluginRegistrant.swift | 2 + packages/flutter_gemma/example/macos/Podfile | 5 + .../flutter_gemma/example/macos/Podfile.lock | 64 ++---- .../macos/Runner.xcodeproj/project.pbxproj | 2 +- packages/flutter_gemma/example/pubspec.lock | 6 +- packages/flutter_gemma/example/web/index.html | 2 +- .../flutter_gemma/ios/flutter_gemma.podspec | 2 +- packages/flutter_gemma/pubspec.yaml | 2 +- .../flutter_gemma_embeddings/CHANGELOG.md | 3 + .../flutter_gemma_embeddings/hook/build.dart | 14 +- .../flutter_gemma_embeddings/pubspec.yaml | 2 +- packages/flutter_gemma_litertlm/CHANGELOG.md | 6 + packages/flutter_gemma_litertlm/README.md | 2 +- .../flutter_gemma_litertlm/hook/build.dart | 14 +- .../lib/src/web/litert_lm_web.dart | 2 +- .../lib/src/web/litert_lm_web_inference.dart | 8 +- packages/flutter_gemma_litertlm/pubspec.yaml | 2 +- website/content/docs/desktop.md | 5 + website/content/docs/genkit.md | 6 +- website/content/docs/migration.md | 6 +- 35 files changed, 1107 insertions(+), 167 deletions(-) create mode 100644 packages/flutter_gemma/example/integration_test/community_models_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/fastvlm_cpu_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/fastvlm_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/internvl35_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/llava_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/phi4_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/qwen25_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/qwen2vl_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/smollm3_smoke_test.dart create mode 100644 packages/flutter_gemma/example/integration_test/smolvlm2_smoke_test.dart diff --git a/CLAUDE.md b/CLAUDE.md index cb10d928..e2dc67d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,7 +138,7 @@ Core has NO pigeon (dropped at the 1.0 cut; its value types are hand-written in - **MediaPipe Web**: v0.10.27, Android/iOS: v0.10.33 - **LiteRT-LM**: native libs from `native-v0.13.1-a` GitHub Release. Android tarball bundles the Qualcomm QNN dispatch stack and Windows tarball bundles Intel NPU dispatch (`LiteRtDispatch.dll` + OpenVino runtime + TBB) for `PreferredBackend.npu` (Qualcomm Snapdragon / Intel LunarLake/PantherLake). MTP (speculative decoding) support for Gemma 4 (#318 MTP crash fixed). (native-v0.13.1-a restores the NPU dispatch libs accidentally omitted from native-v0.13.1 — #155.) - **large_file_handler**: `^0.5.0` (core dep; 0.5.0 declares all 6 platforms — needed for pana platform support + the dart2wasm-clean web graph) -- **Current Version**: core `flutter_gemma` `1.3.1`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.0.4`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.1`; `flutter_gemma_agent` `0.1.0`, `flutter_gemma_builtin_ai` `0.1.0` (new) +- **Current Version**: core `flutter_gemma` `1.3.2`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.2.0`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.3`; `flutter_gemma_agent` `0.1.0`, `flutter_gemma_builtin_ai` `0.1.0` - **0.15.2**: embedding unified on LiteRT C API via Dart FFI on all native platforms (Android + iOS + Desktop). Drops `localagents-rag` JVM dep on Android and the separate TFLite C 0.12.7 tarball on Desktop; `TensorFlowLiteC` pod no longer needed on iOS. Single source of truth for `TaskType.prefix` in Dart, fixes cross-platform embedding drift (#264). ## Platform-Specific Setup diff --git a/packages/flutter_gemma/CHANGELOG.md b/packages/flutter_gemma/CHANGELOG.md index 5afb42a5..47d2ca09 100644 --- a/packages/flutter_gemma/CHANGELOG.md +++ b/packages/flutter_gemma/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.3.2 +- Docs: add community models to the README model list (SmolLM3-3B, Phi-4-mini-reasoning, Qwen2-VL, SmolVLM2, LLaVA-OneVision). +- Docs: bump web `@litert-lm/core` setup snippet to 0.14.0. + ## 1.3.1 - Fix multi-GB background_downloader temp-file leak on Android (#383). - Deterministic download task IDs (sha256 of path triple) — stable across restarts and signed-URL rotation (#383). diff --git a/packages/flutter_gemma/README.md b/packages/flutter_gemma/README.md index a82ff9ec..5e9a2de3 100644 --- a/packages/flutter_gemma/README.md +++ b/packages/flutter_gemma/README.md @@ -7,9 +7,9 @@ [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/flutter_gemma) -**The plugin supports not only Gemma, but also other models. Here's the full list of supported models:** [Gemma 4 E2B/E4B](https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm), [Gemma3n E2B/E4B](https://huggingface.co/google/gemma-3n-E2B-it-litert-preview), [FastVLM 0.5B](https://huggingface.co/litert-community/FastVLM-0.5B), [Gemma-3 1B](https://huggingface.co/litert-community/Gemma3-1B-IT), [Gemma 3 270M](https://huggingface.co/litert-community/gemma-3-270m-it), [FunctionGemma 270M](https://huggingface.co/sasha-denisov/function-gemma-270M-it), [Qwen3 0.6B](https://huggingface.co/litert-community/Qwen3-0.6B), [Qwen 2.5](https://huggingface.co/litert-community/Qwen2.5-1.5B-Instruct), [Phi-4 Mini](https://huggingface.co/litert-community/Phi-4-mini-instruct), [DeepSeek R1](https://huggingface.co/litert-community/DeepSeek-R1-Distill-Qwen-1.5B), [SmolLM 135M](https://huggingface.co/litert-community/SmolLM-135M-Instruct). +**The plugin supports not only Gemma, but also other models. Here's the full list of supported models:** [Gemma 4 E2B/E4B](https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm), [Gemma3n E2B/E4B](https://huggingface.co/google/gemma-3n-E2B-it-litert-preview), [FastVLM 0.5B](https://huggingface.co/litert-community/FastVLM-0.5B), [Gemma-3 1B](https://huggingface.co/litert-community/Gemma3-1B-IT), [Gemma 3 270M](https://huggingface.co/litert-community/gemma-3-270m-it), [FunctionGemma 270M](https://huggingface.co/sasha-denisov/function-gemma-270M-it), [Qwen3 0.6B](https://huggingface.co/litert-community/Qwen3-0.6B), [Qwen 2.5](https://huggingface.co/litert-community/Qwen2.5-1.5B-Instruct), [Phi-4 Mini](https://huggingface.co/litert-community/Phi-4-mini-instruct), [DeepSeek R1](https://huggingface.co/litert-community/DeepSeek-R1-Distill-Qwen-1.5B), [SmolLM 135M](https://huggingface.co/litert-community/SmolLM-135M-Instruct), [SmolLM3 3B](https://huggingface.co/litert-community/SmolLM3-3B), [Phi-4 Mini Reasoning](https://huggingface.co/litert-community/Phi-4-mini-reasoning), [Qwen2-VL 2B](https://huggingface.co/litert-community/Qwen2-VL-2B), [SmolVLM2 500M](https://huggingface.co/litert-community/SmolVLM2-500M), [LLaVA-OneVision 0.5B](https://huggingface.co/litert-community/LLaVA-OneVision-0.5B). -*Note: The flutter_gemma plugin supports Gemma 4 and Gemma3n (with **multimodal vision and audio support**), FastVLM (vision), Gemma-3, FunctionGemma, Qwen3, Qwen 2.5, Phi-4, DeepSeek R1 and SmolLM. Desktop platforms (macOS, Windows, Linux) require `.litertlm` model format. +*Note: The flutter_gemma plugin supports Gemma 4 and Gemma3n (with **multimodal vision and audio support**), FastVLM, Qwen2-VL, SmolVLM2 and LLaVA-OneVision (vision), Gemma-3, FunctionGemma, Qwen3, Qwen 2.5, Phi-4 (incl. Phi-4 Mini Reasoning), DeepSeek R1, SmolLM and SmolLM3. Desktop platforms (macOS, Windows, Linux) require `.litertlm` model format. [Gemma](https://ai.google.dev/gemma) is a family of lightweight, state-of-the art open models built from the same research and technology used to create the Gemini models @@ -31,11 +31,11 @@ There is an example of using: - **Platform Support:** Compatible with iOS, Android, Web, macOS, Windows, and Linux platforms. - **🧩 Modular Packages:** A small `flutter_gemma` core plus opt-in packages — add only the engine (`.litertlm` / `.task`), embeddings, RAG, or agent code your app ships. Register them via one `FlutterGemma.initialize(...)` call. See [MIGRATION.md](MIGRATION.md). - **🖥️ Desktop Support:** Native desktop apps (macOS, Windows, Linux) with GPU acceleration via LiteRT-LM, called directly from Dart through `dart:ffi` — no JVM/JRE bundling. See [DESKTOP_SUPPORT.md](DESKTOP_SUPPORT.md) for details. -- **🖼️ Multimodal Support:** Text + Image input with Gemma 4, Gemma3n, and FastVLM vision models (all platforms incl. Web) +- **🖼️ Multimodal Support:** Text + Image input with Gemma 4, Gemma3n, FastVLM, Qwen2-VL, SmolVLM2, and LLaVA-OneVision vision models (Gemma 4 / Gemma3n on all platforms incl. Web; the `.litertlm`-only VLMs on Android, iOS, and Desktop) - **🎙️ Audio Input:** Record and send audio messages with Gemma 4 and Gemma3n E2B/E4B models (Android, iOS device, macOS/Windows/Linux via LiteRT-LM — not on Web) - **🛠️ Function Calling:** Enable your models to call external functions and integrate with other services (supported by select models) - **🤖 On-device Agent Skills:** Opt-in [`flutter_gemma_agent`](https://pub.dev/packages/flutter_gemma_agent) — give the model `SKILL.md` skills (text / JavaScript / native-intent / MCP) it invokes through the function-calling loop, fully offline. Gallery-compatible. Android, iOS, macOS, Windows (Web not supported yet). -- **🧠 Thinking Mode:** View the reasoning process of Gemma 4, DeepSeek R1, and Qwen3 models with thinking blocks +- **🧠 Thinking Mode:** View the reasoning process of Gemma 4, DeepSeek R1, Qwen3, SmolLM3, and Phi-4 Mini Reasoning models with thinking blocks - **🛑 Stop Generation:** Cancel text generation mid-process on Android, iOS, Web, and Desktop - **⚡ Backend Switching:** Choose between CPU, GPU, and NPU backends per model — CPU/GPU on Android/iOS/Desktop, GPU on Web - **⚙️ NPU Acceleration:** Hardware NPU inference for `.litertlm` models on Qualcomm Snapdragon (Android) and Intel LunarLake/PantherLake (Windows) @@ -123,7 +123,11 @@ The example app offers a curated list of models, each suited for different tasks | **Gemma 4 E4B** | Next-gen multimodal chat — text, image, audio | ✅ | ✅ | ✅ | Multilingual | 4.3GB | | **Gemma3n** | On-device multimodal chat and image analysis | ✅ | ❌ | ✅ | Multilingual | 3-6GB | | **FastVLM 0.5B** | Fast vision-language inference | ❌ | ❌ | ✅ | Multilingual | 0.5GB | +| **Qwen2-VL 2B** | Vision-language chat (image + text) | ❌ | ❌ | ✅ | Multilingual | 1.8GB | +| **SmolVLM2 500M** | Compact vision-language model | ❌ | ❌ | ✅ | Multilingual | 0.36GB | +| **LLaVA-OneVision 0.5B** | Compact vision-language model | ❌ | ❌ | ✅ | Multilingual | 0.83GB | | **Phi-4 Mini** | Advanced reasoning and instruction following | ✅ | ❌ | ❌ | Multilingual | 3.9GB | +| **Phi-4 Mini Reasoning** | Step-by-step reasoning | ❌ | ✅ | ❌ | Multilingual | 2.8GB | | **DeepSeek R1** | High-performance reasoning and code generation | ✅ | ✅ | ❌ | Multilingual | 1.7GB | | **Qwen3 0.6B** | Compact multilingual chat with function calling | ✅ | ✅ | ❌ | Multilingual | 586MB | | **Qwen 2.5** | Strong multilingual chat and instruction following | ✅ | ❌ | ❌ | Multilingual | 0.5-1.6GB | @@ -131,6 +135,7 @@ The example app offers a curated list of models, each suited for different tasks | **Gemma 3 270M** | Ideal for fine-tuning (LoRA) for specific tasks | ❌ | ❌ | ❌ | Multilingual | 0.3GB | | **FunctionGemma 270M** | Specialized for function calling on-device | ✅ | ❌ | ❌ | Multilingual | 284MB | | **SmolLM 135M** | Ultra-compact, resource-constrained devices | ❌ | ❌ | ❌ | English | 135MB | +| **SmolLM3 3B** | Multilingual small LLM with reasoning mode | ❌ | ✅ | ❌ | Multilingual | 2.0GB | | **TranslateGemma 4B** † | Single-shot 55-language translation | ❌ | ❌ | ❌ | 55 languages | 2-4GB | † **TranslateGemma is CPU-only for now.** Google hasn't released a mobile/desktop `.litertlm` bundle (HF discussion [#5](https://huggingface.co/google/translategemma-4b-it/discussions/5) — "no concrete plans"). The example app uses the community-converted bundle from [`barakplasma/translategemma-4b-it-android-task-quantized`](https://huggingface.co/barakplasma/translategemma-4b-it-android-task-quantized), which keeps `EMBEDDING_LOOKUP` weights in float32 for MediaPipe `.task` compatibility. That layout crashes the LiteRT GPU partitioner on Metal/WebGPU across all platforms — tracked upstream at [LiteRT-LM#1748](https://github.com/google-ai-edge/LiteRT-LM/issues/1748). Until Google ships the `litert-lm` quantization CLI, translation runs on CPU only (≈90 s prefill on a 4 B int4 bundle on M-series Macs). @@ -148,7 +153,7 @@ When installing models, you need to specify the correct `ModelType`. Use this ta | **Qwen 3** | `ModelType.qwen3` | Qwen3 0.6B | | **FunctionGemma** | `ModelType.functionGemma` | FunctionGemma 270M IT | | **Phi** | `ModelType.phi` | Phi-4 Mini | -| **General** | `ModelType.general` | FastVLM 0.5B, SmolLM 135M | +| **General** | `ModelType.general` | FastVLM 0.5B, SmolLM 135M, SmolLM3 3B, Phi-4 Mini Reasoning, Qwen2-VL 2B, SmolVLM2 500M, LLaVA-OneVision 0.5B | > **Note**: Gemma 4 uses `ModelType.gemma4` so its native `<\|tool_call>...` tokens are routed through the LiteRT-LM SDK's chat-template path. For Gemma 3 and earlier, keep `ModelType.gemmaIt`. @@ -367,7 +372,7 @@ script(s) for the **engine package(s) you use** to your `web/index.html`. ```html