Overhaul storage

.gitignore

@@ -251,3 +251,6 @@ _metadata
 
 # Stores VSCode versions used for testing VSCode extensions
 .vscode-test
+
+# stupid arch thing
+injectors/desktop/app/app

packages/shelter-storage/.gitignore

@@ -0,0 +1 @@
+dist/

packages/shelter-storage/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@uwu/shelter-storage",
+  "description": "deep proxy & IDB-based storage library for shelter",
+  "version": "1.0.0",
+  "author": "uwu.network",
+  "repository": "github:uwu/shelter",
+  "scripts": {
+    "prepare": "tsc"
+  },
+  "type": "module",
+  "main": "src/index.ts",
+  "module": "src/index.ts",
+  "dependencies": {
+    "solid-js": "1.6.16"
+  }
+}

packages/shelter-storage/src/deep.ts

@@ -0,0 +1,51 @@
+export type ObjPath = (string | symbol)[];
+
+export interface DeepProxyHandler {
+  apply?(path: ObjPath, thisArg: any, args: any[]): any;
+
+  construct?(path: ObjPath, args: any[], newTarget: any): any;
+
+  defineProperty?(path: ObjPath, property: string | symbol, descriptor: PropertyDescriptor): boolean;
+
+  deleteProperty?(path: ObjPath, property: string | symbol): boolean;
+
+  get?(path: ObjPath, property: string | symbol): any;
+
+  getOwnPropertyDescriptor?(path: ObjPath, prop: string | symbol): undefined | PropertyDescriptor;
+
+  has?(path: ObjPath, prop: string | symbol): boolean;
+
+  isExtensible?(path: ObjPath): boolean;
+
+  ownKeys?(path: ObjPath): (string | symbol)[];
+
+  preventExtensions?(path: ObjPath): boolean;
+
+  set?(path: ObjPath, prop: string | symbol, value: any): boolean;
+
+  setPrototypeof?(path: ObjPath, prototype: object | null): boolean;
+}
+
+export function makeDeepProxy<T extends {}>(handler: DeepProxyHandler, init?: T) {
+  const makeProx = <T2>(init: T2, ctxt: ObjPath = []) => {
+    const deepHandler = {
+      get: !handler.get
+        ? undefined
+        : (_, p) => {
+            const gotten = handler.get(ctxt, p);
+
+            return typeof gotten === "function" || (typeof gotten === "object" && gotten !== null)
+              ? makeProx(gotten, [...ctxt, p])
+              : gotten;
+          },
+    };
+
+    for (const k in handler) {
+      deepHandler[k] ??= (_, ...args) => handler[k](ctxt, ...args);
+    }
+
+    return new Proxy(init, deepHandler);
+  };
+
+  return makeProx<T>(init ?? ({} as any)); // as any :D
+}

packages/shelter-storage/src/idb.ts

@@ -0,0 +1,99 @@
+// written using idb-keyval by jake archibald as a guide, code is not copied from there
+// to be on the safe side, idb-keyval license:
+/*
+Copyright 2016, Jake Archibald
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+function promisifyIdbReq<T>(request: IDBRequest<T> | IDBTransaction): Promise<T> {
+  return new Promise((res, rej) => {
+    if ("oncomplete" in request) request.oncomplete = () => res(undefined);
+    else request.onsuccess = () => res(request.result);
+
+    request.onerror = () => rej(request.error);
+  });
+}
+
+export type DbStore = <T>(txm: IDBTransactionMode, cb: (s: IDBObjectStore) => T | PromiseLike<T>) => Promise<T>;
+
+// all DBs must have their open()s start before the db actually opens
+let storesToAdd: string[] = [];
+let openProm: Promise<IDBDatabase>;
+export async function open(store: string): Promise<DbStore> {
+  const makeDbStore =
+    (db: Promise<IDBDatabase>): DbStore =>
+    (txm, cb) =>
+      db.then((d) => cb(d.transaction(store, txm).objectStore(store)));
+
+  storesToAdd.push(store);
+
+  if (storesToAdd.length <= 1) {
+    let res, rej;
+    openProm = new Promise<IDBDatabase>((r, j) => ((res = r), (rej = j)));
+
+    // check if we *need* to upgrade:
+    const req1 = indexedDB.open("shelter");
+    const sns = (await promisifyIdbReq(req1)).objectStoreNames;
+    let needUpgrade = false;
+    for (const store of storesToAdd) if (!sns.contains(store)) needUpgrade = true;
+
+    //if (needUpgrade)
+    const req = indexedDB.open("shelter", Date.now());
+    req.onupgradeneeded = () => {
+      for (const s of storesToAdd) if (!req.result.objectStoreNames.contains(s)) req.result.createObjectStore(s);
+
+      storesToAdd = [];
+    };
+    openProm = promisifyIdbReq(req);
+  }
+
+  await openProm;
+  return makeDbStore(promisifyIdbReq(indexedDB.open("shelter")));
+}
+
+export function get<T>(key: IDBValidKey, store: DbStore): Promise<T> {
+  return store("readonly", (st) => promisifyIdbReq(st.get(key)));
+}
+
+export function getMult<T>(keys: IDBValidKey[], store: DbStore): Promise<T[]> {
+  return store("readonly", (st) => Promise.all(keys.map((k) => promisifyIdbReq(st.get(k)))));
+}
+
+export function set<T>(key: IDBValidKey, value: T, store: DbStore): Promise<void> {
+  return store("readwrite", (st) => promisifyIdbReq((st.put(value, key), st.transaction)));
+}
+
+export function setMult(entries: [IDBValidKey, any][], store: DbStore): Promise<void> {
+  return store("readwrite", (st) => promisifyIdbReq((entries.forEach((e) => st.put(e[1], e[0])), st.transaction)));
+}
+
+export function remove(key: IDBValidKey, store: DbStore): Promise<void> {
+  return store("readwrite", (st) => promisifyIdbReq((st.delete(key), st.transaction)));
+}
+
+export function removeMult(keys: IDBValidKey[], store: DbStore): Promise<void> {
+  return store("readwrite", (st) => promisifyIdbReq((keys.forEach((e) => st.delete(e)), st.transaction)));
+}
+
+export function keys(store: DbStore) {
+  return store("readonly", (st) => promisifyIdbReq(st.getAllKeys()));
+}
+
+export function entries(store: DbStore) {
+  return store("readonly", (st) =>
+    Promise.all([promisifyIdbReq(st.getAllKeys()), promisifyIdbReq(st.getAll())]).then(([k, v]) =>
+      k.map((key, i) => [key, v[i]] as const),
+    ),
+  );
+}

packages/shelter-storage/src/idbStore.ts

@@ -0,0 +1,159 @@
+import { batch, untrack } from "solid-js";
+import { makeDeepProxy } from "./deep";
+import { DbStore, entries, keys, open, remove, set as dbSet } from "./idb";
+import { delKey, getNode, getValue, has, makeRoot, set } from "./signalTree";
+
+const symWait = Symbol();
+const symReady = Symbol();
+const symDb = Symbol();
+const symSig = Symbol();
+
+export interface IdbStore<T> {
+  [_: string]: T;
+
+  [symWait]: (cb: () => void) => void;
+  [symReady]: boolean;
+  [symDb]: DbStore; //IDBPDatabase<any>;
+  [symSig]: () => Record<string, T>;
+}
+
+let getdbprom: Promise<unknown>;
+
+// todo: can't bootstrap stores. oops.
+const getDb = async (store: string) => open(store);
+
+export const idbStore = <T = any>(name: string) => {
+  const tree = makeRoot();
+  let store: DbStore;
+  let inited = false;
+  let modifiedKeys = new Set<string>();
+
+  // queues callbacks for when the db loads
+  const waitQueue: (() => void)[] = [];
+  const waitInit = (cb: () => void) => void (inited ? cb() : waitQueue.push(cb));
+
+  // get all entries of db to setup initial state
+  getDb(name)
+    .then((s) => {
+      store = s;
+      return entries(s);
+    })
+    .then((e) => {
+      // if a node exists but wasn't modified (get), set it from db
+      // if a node exists and was modified, (set, delete) leave it be
+      // if a node does not exist, create it from db
+      for (const [_k, v] of e) {
+        // idbvalidkey is more permissive than keyof {}
+        const k = _k as string;
+
+        if (k in tree.children) {
+          if (!modifiedKeys.has(k)) set(tree, [k], v);
+        } else {
+          set(tree, [k], v);
+        }
+      }
+
+      inited = true;
+      waitQueue.forEach((cb) => cb());
+      waitQueue.length = 0;
+    });
+
+  return makeDeepProxy({
+    get(path, p) {
+      // internal things
+      if (p === symWait) return waitInit;
+      if (p === symReady) return inited;
+      if (p === symDb) return store;
+      if (p === symSig) return () => tree.sig[0]();
+
+      // etc
+      //if (typeof p === "symbol") throw new Error("cannot index idb store with a symbol");
+
+      //return getNode(tree, [...(path as string[]), p])?.sig[0]();
+      return getValue(tree, [...(path as string[]), p]);
+    },
+
+    set(path, p, v) {
+      if (typeof p === "symbol") throw new Error("cannot index idb store with a symbol");
+
+      const resolvedPath = [...(path as string[]), p];
+      const topLevelPath = resolvedPath[0];
+      modifiedKeys.add(topLevelPath);
+
+      set(tree, resolvedPath, v);
+
+      waitInit(async () => {
+        const val = untrack(getNode(tree, [topLevelPath]).sig[0]);
+        await dbSet(topLevelPath, val, store);
+      });
+
+      return true;
+    },
+
+    deleteProperty(path, p) {
+      if (typeof p === "symbol") throw new Error("cannot index idb store with a symbol");
+
+      const resolvedPath = [...(path as string[]), p];
+      const topLevelPath = resolvedPath[0];
+      modifiedKeys.add(topLevelPath);
+
+      delKey(tree, resolvedPath);
+      waitInit(async () => {
+        if (path.length === 0) await remove(p, store);
+        else {
+          const val = untrack(getNode(tree, [topLevelPath]).sig[0]);
+          await dbSet(topLevelPath, val, store);
+        }
+      });
+
+      return true;
+    },
+
+    has: (path, p) => has(tree, [...path, p] as string[]),
+
+    ownKeys(path) {
+      const node = getNode(tree, path as string[]);
+      return Reflect.ownKeys(
+        /*node.type === "object" || node.type === "root" /!*|| node.type === "array"*!/ ? node.children :*/ untrack(
+          node.sig[0],
+        ),
+      );
+    },
+
+    // without this, properties are not enumerable! (object.keys wouldn't work)
+    getOwnPropertyDescriptor: (path, p) => {
+      const parentNode = getNode(tree, path as string[]);
+      if (!parentNode) return undefined;
+
+      /*if ((parentNode.type === "object" || parentNode.type === "root") && p in parentNode.children)
+        return ({
+          value: null, // this should never be directly accessed, `get` should be used.
+          enumerable: true,
+          configurable: true,
+          writable: true,
+        });
+      else*/
+      return Reflect.getOwnPropertyDescriptor(untrack(parentNode.sig[0]), p);
+    },
+  }) as IdbStore<T>;
+};
+
+// stuff like this is necessary when you *need* to have gets return persisted values as well as newly set ones
+
+/** if the store is or is not yet connected to IDB */
+export const isInited = (store: IdbStore<unknown>) => store[symReady];
+/** waits for the store to connect to IDB, then runs the callback (if connected, synchronously runs the callback now) */
+export const whenInited = (store: IdbStore<unknown>, cb: () => void) => store[symWait](cb);
+/** returns a promise that resolves when the store is connected to IDB (if connected, resolves instantly) */
+export const waitInit = (store: IdbStore<unknown>) => new Promise<void>((res) => whenInited(store, res));
+
+/** sets default values for the store. these only apply once the store connects to IDB to prevent overwriting persist */
+export const defaults = <T = any>(store: IdbStore<T>, fallbacks: Record<string, T>) =>
+  whenInited(store, () =>
+    batch(() => {
+      for (const k in fallbacks) if (!(k in store)) store[k] = fallbacks[k];
+    }),
+  );
+
+/** gets a signal containing the whole store as an object */
+export const signalOf = <T = any>(store: IdbStore<T>): (() => Record<string, T>) => store[symSig];

packages/shelter-storage/src/index.ts

@@ -0,0 +1,6 @@
+export * from "./idbStore";
+export * from "./deep";
+
+import * as idb from "./idb";
+import * as signalTree from "./signalTree";
+export { idb, signalTree };