Dexie Resilience

api/web/src/base/overlay-class.ts

@@ -242,6 +242,10 @@ export default class Overlay {
         }
 
         for (const l of this.styles) {
+            // A previous partially-failed initOverlays() attempt may have
+            // already registered this layer; addLayer throws on duplicate ids.
+            if (mapStore.map.getLayer(l.id)) continue;
+
             if (before) {
                 mapStore.map.addLayer(l, before);
             } else {

api/web/src/components/CloudTAK/Map.vue

@@ -504,6 +504,7 @@ import { useAppStore } from '../../stores/app.ts';
 import { DrawToolMode } from '../../stores/modules/draw.ts';
 import { useFloatStore } from '../../stores/float.ts';
 import { liveQuery } from 'dexie';
+import { isTransientDbError } from '../../database.ts';
 import Upload from '../util/Upload.vue';
 import { stdurl } from '../../std.ts';
 import ProfileConfig from '../../base/profile.ts';
@@ -637,18 +638,29 @@ const mapSideOffset = computed(() => {
     return Math.max(mapStore.toastOffset.x - 10, 0);
 });
 
+function onWindowError(evt: ErrorEvent) {
+    console.error(evt);
+    evt.preventDefault();
+
+    // Prefer the original error so its name survives; transient
+    // IndexedDB failures (WebKit invalidating the connection during an
+    // iOS suspend) are recovered by withDbRetry and must not modal.
+    const err = evt.error instanceof Error ? evt.error : new Error(evt.message);
+    if (isTransientDbError(err)) return;
+
+    emit('err', err);
+}
+
+function onWindowResize() {
+    height.value = window.innerHeight;
+    width.value = window.innerWidth;
+}
+
 onMounted(async () => {
     // ensure uncaught errors in the stack are captured into vue context
-    window.addEventListener('error', (evt) => {
-        console.error(evt);
-        evt.preventDefault();
-        emit('err', new Error(evt.message));
-    });
+    window.addEventListener('error', onWindowError);
 
-    window.addEventListener('resize', () => {
-        height.value = window.innerHeight;
-        width.value = window.innerWidth;
-    });
+    window.addEventListener('resize', onWindowResize);
 
     if (!mapRef.value) throw new Error('Map Element could not be found - Please refresh the page and try again');
     await mapStore.init(mapRef.value);
@@ -711,6 +723,8 @@ onMounted(async () => {
 });
 
 onBeforeUnmount(() => {
+    window.removeEventListener('error', onWindowError);
+    window.removeEventListener('resize', onWindowResize);
     inviteChannel?.close();
     void mapStore.destroy();
 });

api/web/src/database.ts

@@ -329,6 +329,11 @@ db.version(2).stores({
 
 let reopenPromise: Promise<void> | null = null;
 
+// Increments on every successful (re)open so callers can tell whether the
+// connection they observed failing has already been replaced.
+let connectionGeneration = 0;
+let forceReopenPromise: Promise<void> | null = null;
+
 // An IndexedDB open issued while a page is unloading leaves WKWebView's
 // database process holding an orphaned request that deadlocks the next
 // page's first IndexedDB operation until the app is fully restarted. Close
@@ -366,6 +371,7 @@ export async function ensureDatabase(): Promise<void> {
 
                 try {
                     await db.open();
+                    connectionGeneration++;
                     return;
                 } catch (err) {
                     if (db.isOpen()) return;
@@ -383,6 +389,41 @@ export async function ensureDatabase(): Promise<void> {
     return reopenPromise;
 }
 
+/**
+ * Close and reopen the IndexedDB connection.
+ *
+ * WebKit on iOS can invalidate the underlying connection while the app is
+ * suspended without firing a close event, leaving Dexie reporting an open —
+ * but dead — connection whose every request fails with UnknownError
+ * ("attempt to get records from the database without an in-progress
+ * transaction"). ensureDatabase() is a no-op in that state, so recovery
+ * requires an explicit close first.
+ *
+ * Pass the generation observed before the failing operation so a connection
+ * another caller already replaced is not needlessly closed again.
+ */
+export async function forceReopenDatabase(sinceGeneration = connectionGeneration): Promise<void> {
+    if (shuttingDown) return;
+
+    if (!forceReopenPromise) {
+        if (sinceGeneration !== connectionGeneration) return ensureDatabase();
+
+        forceReopenPromise = (async () => {
+            try {
+                db.close();
+            } catch (err) {
+                console.warn('Failed to close database before reopen', err);
+            }
+
+            await ensureDatabase();
+        })().finally(() => {
+            forceReopenPromise = null;
+        });
+    }
+
+    return forceReopenPromise;
+}
+
 const TRANSIENT_DB_ERROR_NAMES = new Set([
     'AbortError',
     'DatabaseClosedError',
@@ -399,7 +440,11 @@ const TRANSIENT_DB_ERROR_MESSAGES = [
     'transaction aborted',
     'objectstore',
     'connection is closing',
-    'premature commit'
+    'premature commit',
+    // WebKit UnknownError after iOS suspends the app mid-transaction; matched
+    // on message because wrappers (window.onerror, String(reason)) lose the
+    // DOMException name.
+    'in-progress transaction'
 ];
 
 db.on('close', () => {
@@ -418,14 +463,26 @@ export async function withDbRetry<T>(fn: () => Promise<T>, attempts = 4): Promis
     let lastError: unknown;
 
     for (let attempt = 0; attempt < attempts; attempt++) {
+        let generation = connectionGeneration;
+
         try {
             await ensureDatabase();
+            generation = connectionGeneration;
             return await fn();
         } catch (err) {
             lastError = err;
             if (!isTransientDbError(err)) throw err;
 
-            await ensureDatabase();
+            // Dexie can still report the connection open after WebKit has
+            // invalidated it (iOS suspend fires no close event), in which
+            // case ensureDatabase() alone is a no-op and every retry would
+            // fail identically — force a real close+reopen.
+            try {
+                await forceReopenDatabase(generation);
+            } catch (reopenErr) {
+                console.warn('Failed to reopen database during retry', reopenErr);
+            }
+
             await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
         }
     }

api/web/src/stores/map.ts

@@ -39,7 +39,7 @@ import { CloudTAKTransferHandler } from '../base/handler.ts';
 import ProfileConfig from '../base/profile.ts';
 import Config from '../base/config.ts';
 import { isNativePlatform, addBackgroundStateListener } from '../base/capacitor.ts';
-import { ensureDatabase } from '../database.ts';
+import { db, withDbRetry } from '../database.ts';
 
 import type { ProfileOverlay, Basemap, Feature } from '../types.ts';
 import type { LngLat, LngLatLike, Point, MapMouseEvent, MapTouchEvent, MapGeoJSONFeature, GeoJSONSource } from 'maplibre-gl';
@@ -83,7 +83,6 @@ export const useMapStore = defineStore('cloudtak', {
         _bottomBar?: unknown;
 
         _removeOrientationListener?: () => Promise<void>;
-        _boundOnVisibilityChange?: () => Promise<void>;
         _removeBackgroundStateListener?: () => void;
         _removePushTokenListener?: () => void;
 
@@ -422,7 +421,6 @@ export const useMapStore = defineStore('cloudtak', {
                 await this._removeOrientationListener();
                 this._removeOrientationListener = undefined;
             }
-            if (this._boundOnVisibilityChange) document.removeEventListener('visibilitychange', this._boundOnVisibilityChange);
             if (this._removeBackgroundStateListener) {
                 this._removeBackgroundStateListener();
                 this._removeBackgroundStateListener = undefined;
@@ -703,6 +701,37 @@ export const useMapStore = defineStore('cloudtak', {
 
             return sub;
         },
+
+        /**
+         * Recover shared state after the app returns to the foreground. iOS
+         * can invalidate IndexedDB connections (without firing a close event)
+         * and drop the WebSocket while the WebView is suspended, so probe the
+         * main-thread and worker database connections — withDbRetry force-
+         * reopens a dead one — then reconnect the WebSocket and re-render CoTs.
+         */
+        resumeFromBackground: async function(): Promise<void> {
+            if (!this._worker || !(await this.worker.initialized)) return;
+
+            try {
+                await withDbRetry(() => db.kv.get('serverUrl'));
+            } catch (err) {
+                console.error('Failed to reopen IndexedDB on resume:', err);
+            }
+
+            try {
+                await this.worker.resume();
+            } catch (err) {
+                console.error('Failed to recover worker database on resume:', err);
+            }
+
+            const isOpen = await this.worker.conn.isOpen;
+            if (!isOpen) {
+                console.log('Resumed with closed connection, reconnecting...');
+                await this.worker.conn.reconnect(await this.worker.username);
+            }
+
+            await this.updateCOT();
+        },
         init: async function(container: HTMLElement) {
             // Start the worker here rather than in state() so that std.ts
             // inside the worker resolves serverUrl from KV only after
@@ -714,27 +743,6 @@ export const useMapStore = defineStore('cloudtak', {
 
             this.container = container;
 
-            this._boundOnVisibilityChange = async (): Promise<void> => {
-                if (document.hidden) return;
-                if (!(await this.worker.initialized)) return;
-
-                // Proactively reopen the main-thread IndexedDB connection.
-                // WebKit may have force-closed it while the app was backgrounded.
-                try {
-                    await ensureDatabase();
-                } catch (err) {
-                    console.error('Failed to reopen IndexedDB on resume:', err);
-                }
-
-                const isOpen = await this.worker.conn.isOpen;
-                if (!isOpen) {
-                    console.log('Tab became visible with closed connection, reconnecting...');
-                    await this.worker.conn.reconnect(await this.worker.username);
-                }
-
-                await this.updateCOT();
-            };
-
             this._removeOrientationListener = await deviceStore.orientation.addListener((heading) => {
                 // Drive the self-location puck's heading cone regardless of
                 // whether the map itself is being rotated to match.
@@ -747,14 +755,21 @@ export const useMapStore = defineStore('cloudtak', {
                     this.map.setBearing(heading);
                 }
             });
-            document.addEventListener('visibilitychange', this._boundOnVisibilityChange);
 
             // Track foreground/background transitions using a native-reliable
-            // signal so background location reporting (submitLocationHttp) is
-            // gated correctly on iOS, where document.hidden is unreliable.
+            // signal (Capacitor appStateChange; document.hidden is unreliable
+            // on iOS). Background location reporting (submitLocationHttp) is
+            // gated on it, and returning to the foreground must recover the
+            // IndexedDB connections and WebSocket iOS severed during suspend.
             this.isBackgrounded = false;
             this._removeBackgroundStateListener = await addBackgroundStateListener((isBackgrounded) => {
                 this.isBackgrounded = isBackgrounded;
+
+                if (!isBackgrounded) {
+                    this.resumeFromBackground().catch((err) => {
+                        console.error('Failed to recover after returning to foreground:', err);
+                    });
+                }
             });
 
             const { value: token } = await Preferences.get({ key: 'token' });
@@ -969,24 +984,8 @@ export const useMapStore = defineStore('cloudtak', {
             map.addControl(routingControl);
             (map as mapgl.Map & { _routingControl?: RoutingControl })._routingControl = routingControl;
 
-            map.once('idle', async () => {
-                const displayProjection = await ProfileConfig.get('display_projection');
-
-                if (displayProjection && displayProjection.value === 'globe') {
-                    map.setProjection({ type: "globe" });
-                }
-
-                void this.icons.hydrate()
-                    .catch((error: unknown) => {
-                        console.error('Failed to hydrate iconsets after map idle', error);
-                    });
-
-                await this.initOverlays();
-
-                this.timer = setInterval(async () => {
-                    if (!this.map) return;
-                    await this.refresh();
-                }, 500);
+            map.once('idle', () => {
+                void this.onMapReady();
             });
 
             // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -1070,8 +1069,51 @@ export const useMapStore = defineStore('cloudtak', {
                 console.warn('Failed to submit background location via HTTP', err);
             }
         },
-        initOverlays: async function() {
-            if (!this.map) throw new Error('Cannot initLayers before map has loaded');
+        /**
+         * Runs once the initial style settles. isLoaded — which dismisses the
+         * map loading screen — is only set at the end of initOverlays(), so a
+         * single failure there (a timed-out request or a dead IndexedDB
+         * connection on a low-bandwidth native resume) must not strand the
+         * user on the loading screen: overlay loading is retried with capped
+         * backoff until it succeeds or the map is torn down.
+         */
+        onMapReady: async function(): Promise<void> {
+            try {
+                const displayProjection = await ProfileConfig.get('display_projection');
+
+                if (displayProjection && displayProjection.value === 'globe') {
+                    this.map.setProjection({ type: "globe" });
+                }
+            } catch (err) {
+                console.error('Failed to apply saved display projection', err);
+            }
+
+            void this.icons.hydrate()
+                .catch((error: unknown) => {
+                    console.error('Failed to hydrate iconsets after map idle', error);
+                });
+
+            this.registerMapListeners();
+
+            for (let attempt = 0; this._map && !this.isLoaded; attempt++) {
+                try {
+                    await this.initOverlays();
+                } catch (err) {
+                    const delay = Math.min(2000 * 2 ** attempt, 30000);
+                    console.error(`Failed to load overlays (attempt ${attempt + 1}), retrying in ${delay}ms:`, err);
+                    await new Promise((resolve) => setTimeout(resolve, delay));
+                }
+            }
+
+            if (!this._map) return;
+
+            this.timer = setInterval(async () => {
+                if (!this.map) return;
+                await this.refresh();
+            }, 500);
+        },
+        registerMapListeners: function() {
+            if (!this.map) throw new Error('Cannot registerMapListeners before map has loaded');
 
             const map: mapgl.Map = this.map as mapgl.Map;
 
@@ -1301,6 +1343,19 @@ export const useMapStore = defineStore('cloudtak', {
                 }
             });
 
+        },
+        /**
+         * Load every overlay (basemaps, profile overlays, missions, the
+         * internal Map Features store) into the map and mark it isLoaded.
+         * Must be re-runnable: onMapReady() retries it after a failure, and
+         * source/layer registration is guarded against duplicates from a
+         * previous partial attempt.
+         */
+        initOverlays: async function() {
+            if (!this.map) throw new Error('Cannot initOverlays before map has loaded');
+
+            const map: mapgl.Map = this.map as mapgl.Map;
+
             OverlayManager.clearLoaded();
             const profileOverlays = await OverlayManager.list({ localFirst: true });