refactor(vector-search): replace sqlite-vec with hnswlib-wasm

Replace native SQLite extension (sqlite-vec) with pure WASM-based HNSW
implementation for cross-platform compatibility. Removes complex dylib
building and custom SQLite loading logic, simplifying deployment and
maintenance. Vector operations now use hnswlib-wasm with persistent
on-disk indexes stored alongside SQLite databases. BREAKING CHANGE:
removes sqlite-vec dependency and macOS native dylib bundles.

Author: tickernelz

.github/workflows/release.yml

@@ -5,174 +5,16 @@ on:
     tags:
       - 'v*'
 
-env:
-  SQLITE_VEC_VERSION: "0.1.7-alpha.8"
-
 jobs:
-  build-sqlite:
-    runs-on: ${{ matrix.runs-on }}
-    strategy:
-      matrix:
-        include:
-          - arch: arm64
-            runs-on: macos-15
-          - arch: x64
-            runs-on: macos-15-intel
-    
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Get latest SQLite version
-        id: sqlite-version
-        run: |
-          DOWNLOAD_PATH=$(curl -sL https://sqlite.org/download.html | grep -o '[0-9]*/sqlite-amalgamation-[0-9]*\.zip' | head -1)
-          VERSION=$(echo "$DOWNLOAD_PATH" | grep -o 'sqlite-amalgamation-[0-9]*' | sed 's/sqlite-amalgamation-//')
-          YEAR=$(echo "$DOWNLOAD_PATH" | cut -d'/' -f1)
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
-          echo "year=$YEAR" >> $GITHUB_OUTPUT
-
-      - name: Check existing dylib
-        id: check
-        run: |
-          DYLIB_PATH="native/darwin-${{ matrix.arch }}/libsqlite3.dylib"
-          
-          if [ ! -f "$DYLIB_PATH" ]; then
-            echo "needs-update=true" >> $GITHUB_OUTPUT
-            echo "Dylib not found, will build"
-          else
-            EXISTING_SIZE=$(stat -f%z "$DYLIB_PATH" 2>/dev/null || stat -c%s "$DYLIB_PATH" 2>/dev/null || echo "0")
-            if [ "$EXISTING_SIZE" -lt 1400000 ]; then
-              echo "needs-update=true" >> $GITHUB_OUTPUT
-              echo "Dylib too small (likely missing sqlite-vec), will rebuild"
-            else
-              if strings "$DYLIB_PATH" | grep -q "vec_version"; then
-                echo "needs-update=false" >> $GITHUB_OUTPUT
-                echo "Dylib exists with sqlite-vec"
-              else
-                echo "needs-update=true" >> $GITHUB_OUTPUT
-                echo "Dylib missing sqlite-vec, will rebuild"
-              fi
-            fi
-          fi
-
-      - name: Download SQLite and sqlite-vec
-        if: steps.check.outputs.needs-update == 'true'
-        run: |
-          curl -LO "https://sqlite.org/${{ steps.sqlite-version.outputs.year }}/sqlite-amalgamation-${{ steps.sqlite-version.outputs.version }}.zip"
-          unzip "sqlite-amalgamation-${{ steps.sqlite-version.outputs.version }}.zip"
-          
-          curl -LO "https://github.com/asg017/sqlite-vec/releases/download/v${{ env.SQLITE_VEC_VERSION }}/sqlite-vec-${{ env.SQLITE_VEC_VERSION }}-amalgamation.zip"
-          unzip "sqlite-vec-${{ env.SQLITE_VEC_VERSION }}-amalgamation.zip"
-
-      - name: Build combined dylib
-        if: steps.check.outputs.needs-update == 'true'
-        run: |
-          SQLITE_DIR="sqlite-amalgamation-${{ steps.sqlite-version.outputs.version }}"
-          
-          gcc -c "$SQLITE_DIR/sqlite3.c" -o sqlite3.o \
-            -DSQLITE_ENABLE_FTS5 \
-            -DSQLITE_ENABLE_RTREE \
-            -DSQLITE_ENABLE_GEOPOLY
-          
-          gcc -c sqlite-vec.c -o sqlite-vec.o \
-            -I"$SQLITE_DIR" \
-            -DSQLITE_CORE \
-            -DSQLITE_VEC_STATIC
-          
-          cat > vec_init.c << 'INIT_EOF'
-          #include "sqlite3.h"
-          extern int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
-          __attribute__((constructor))
-          static void vec_auto_register(void) {
-              sqlite3_auto_extension((void(*)(void))sqlite3_vec_init);
-          }
-          INIT_EOF
-          
-          gcc -c vec_init.c -o vec_init.o -I"$SQLITE_DIR"
-          
-          gcc -dynamiclib \
-            -o libsqlite3.dylib \
-            sqlite3.o sqlite-vec.o vec_init.o \
-            -install_name @rpath/libsqlite3.dylib \
-            -lpthread -ldl -lm
-
-      - name: Verify dylib
-        if: steps.check.outputs.needs-update == 'true'
-        run: |
-          otool -D libsqlite3.dylib
-          file libsqlite3.dylib
-          ls -la libsqlite3.dylib
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: libsqlite3-darwin-${{ matrix.arch }}
-          path: libsqlite3.dylib
-          retention-days: 1
-          if-no-files-found: ignore
-
-  commit-dylib:
-    needs: build-sqlite
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    outputs:
-      changed: ${{ steps.update.outputs.changed }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: main
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-
-      - name: Update dylib files
-        id: update
-        run: |
-          CHANGED="false"
-          
-          if [ -f "artifacts/libsqlite3-darwin-arm64/libsqlite3.dylib" ]; then
-            mkdir -p native/darwin-arm64
-            cp artifacts/libsqlite3-darwin-arm64/libsqlite3.dylib native/darwin-arm64/
-            CHANGED="true"
-            echo "Updated arm64 dylib"
-          fi
-          
-          if [ -f "artifacts/libsqlite3-darwin-x64/libsqlite3.dylib" ]; then
-            mkdir -p native/darwin-x64
-            cp artifacts/libsqlite3-darwin-x64/libsqlite3.dylib native/darwin-x64/
-            CHANGED="true"
-            echo "Updated x64 dylib"
-          fi
-          
-          echo "changed=$CHANGED" >> $GITHUB_OUTPUT
-
-      - name: Commit and push
-        if: steps.update.outputs.changed == 'true'
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add native/
-          git commit -m "chore: update SQLite+sqlite-vec dylib [skip ci]"
-          git push origin main
-
   release:
-    needs: [build-sqlite, commit-dylib]
     runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: main
           fetch-depth: 0
 
-      - name: Pull latest changes
-        run: git pull origin main
-
       - uses: oven-sh/setup-bun@v2
 
       - run: bun install
@@ -192,16 +34,7 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      - name: Prepare release assets
-        run: |
-          mkdir -p release-assets
-          cp native/darwin-arm64/libsqlite3.dylib release-assets/libsqlite3-darwin-arm64.dylib
-          cp native/darwin-x64/libsqlite3.dylib release-assets/libsqlite3-darwin-x64.dylib
-
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v2
         with:
           generate_release_notes: true
-          files: |
-            release-assets/libsqlite3-darwin-arm64.dylib
-            release-assets/libsqlite3-darwin-x64.dylib

README.md

@@ -20,7 +20,7 @@ A persistent memory system for AI coding agents that enables long-term context r
 
 ## Core Features
 
-Local vector database with SQLite, persistent project memories, automatic user profile learning, unified memory-prompt timeline, full-featured web UI, intelligent prompt-based memory extraction, multi-provider AI support (OpenAI, Anthropic), 12+ local embedding models, smart deduplication, and built-in privacy protection.
+Local vector database with SQLite + HNSW (hnswlib-wasm), persistent project memories, automatic user profile learning, unified memory-prompt timeline, full-featured web UI, intelligent prompt-based memory extraction, multi-provider AI support (OpenAI, Anthropic), 12+ local embedding models, smart deduplication, and built-in privacy protection.
 
 ## Getting Started
 
@@ -32,7 +32,7 @@ Add to your OpenCode configuration at `~/.config/opencode/opencode.json`:
 }
 ```
 
-The plugin downloads automatically on next startup. On macOS, the plugin automatically downloads a compatible SQLite library during installation. If the download fails, install Homebrew SQLite as fallback: `brew install sqlite`
+The plugin downloads automatically on next startup. No additional dependencies required - vector search is powered by pure WASM for cross-platform compatibility.
 
 ## Usage Examples
 

bun.lock

@@ -1,6 +1,6 @@
 {
   "name": "opencode-mem",
-  "version": "2.10.0",
+  "version": "2.11.0",
   "description": "OpenCode plugin that gives coding agents persistent memory using local vector database",
   "type": "module",
   "main": "dist/plugin.js",
@@ -36,6 +36,7 @@
     "@opencode-ai/plugin": "^1.0.162",
     "@xenova/transformers": "^2.17.2",
     "franc-min": "^6.2.0",
+    "hnswlib-wasm": "^0.8.2",
     "iso-639-3": "^3.0.1"
   },
   "devDependencies": {
@@ -54,7 +55,6 @@
   },
   "files": [
     "dist",
-    "native",
     "package.json"
   ],
   "lint-staged": {

native/darwin-arm64/.gitkeep

@@ -327,7 +327,7 @@ export async function handleAddMemory(data: {
       metadata: JSON.stringify({ source: "api" }),
     };
     const db = connectionManager.getConnection(shard.dbPath);
-    vectorSearch.insertVector(db, record);
+    vectorSearch.insertVector(db, record, shard);
     shardManager.incrementVectorCount(shard.id);
     return { success: true, data: { id } };
   } catch (error) {
@@ -352,7 +352,7 @@ export async function handleDeleteMemory(
           const linkedPromptId = metadata?.promptId;
           if (linkedPromptId) userPromptManager.deletePrompt(linkedPromptId);
         }
-        vectorSearch.deleteVector(db, id);
+        await vectorSearch.deleteVector(db, id, shard);
         shardManager.decrementVectorCount(shard.id);
         return {
           success: true,
@@ -406,7 +406,7 @@ export async function handleUpdateMemory(
     }
     if (!foundShard || !existingMemory) return { success: false, error: "Memory not found" };
     const db = connectionManager.getConnection(foundShard.dbPath);
-    vectorSearch.deleteVector(db, id);
+    await vectorSearch.deleteVector(db, id, foundShard);
     shardManager.decrementVectorCount(foundShard.id);
 
     const newContent = data.content || existingMemory.content;
@@ -436,7 +436,7 @@ export async function handleUpdateMemory(
       projectName: existingMemory.project_name,
       gitRepoUrl: existingMemory.git_repo_url,
     };
-    vectorSearch.insertVector(db, updatedRecord);
+    vectorSearch.insertVector(db, updatedRecord, foundShard);
     shardManager.incrementVectorCount(foundShard.id);
     return { success: true };
   } catch (error) {
@@ -497,7 +497,7 @@ export async function handleSearch(
       const shards = shardManager.getAllShards(scope, hash);
       for (const shard of shards) {
         try {
-          const results = vectorSearch.searchInShard(shard, queryVector, tag, pageSize * 2);
+          const results = await vectorSearch.searchInShard(shard, queryVector, tag, pageSize * 2);
           memoryResults.push(...results);
         } catch (error) {
           log("Shard search error", { shardId: shard.id, error: String(error) });
@@ -520,7 +520,12 @@ export async function handleSearch(
         const shards = shardManager.getAllShards(scope, hash);
         for (const shard of shards) {
           try {
-            const results = vectorSearch.searchInShard(shard, queryVector, containerTag, pageSize);
+            const results = await vectorSearch.searchInShard(
+              shard,
+              queryVector,
+              containerTag,
+              pageSize
+            );
             memoryResults.push(...results);
           } catch (error) {
             log("Shard search error", { shardId: shard.id, error: String(error) });
@@ -1053,21 +1058,10 @@ export async function handleRunTagMigrationBatch(
           m.id
         );
 
-        db.prepare("DELETE FROM vec_memories WHERE memory_id = ?").run(m.id);
-        db.prepare("INSERT INTO vec_memories (memory_id, embedding) VALUES (?, ?)").run(
-          m.id,
-          vectorBuffer
-        );
-
-        if (currentTags.length > 0) {
-          const tagsVector = await embeddingService.embedWithTimeout(currentTags.join(", "));
-          const tagsVectorBuffer = new Uint8Array(tagsVector.buffer);
-          db.prepare("DELETE FROM vec_tags WHERE memory_id = ?").run(m.id);
-          db.prepare("INSERT INTO vec_tags (memory_id, embedding) VALUES (?, ?)").run(
-            m.id,
-            tagsVectorBuffer
-          );
-        }
+        const index = vectorSearch
+          .getIndexManager()
+          .getIndex(shard.scope, shard.scopeHash, shard.shardIndex);
+        await index.insert(m.id, vector);
 
         migrationProgress.processed++;
         batchProcessed++;

native/darwin-arm64/libsqlite3.dylib

@@ -195,7 +195,7 @@ export class LocalMemoryClient {
       };
 
       const db = connectionManager.getConnection(shard.dbPath);
-      vectorSearch.insertVector(db, record);
+      vectorSearch.insertVector(db, record, shard);
       shardManager.incrementVectorCount(shard.id);
 
       return { success: true as const, id };
@@ -219,7 +219,7 @@ export class LocalMemoryClient {
         const memory = vectorSearch.getMemoryById(db, memoryId);
 
         if (memory) {
-          vectorSearch.deleteVector(db, memoryId);
+          await vectorSearch.deleteVector(db, memoryId, shard);
           shardManager.decrementVectorCount(shard.id);
           return { success: true };
         }