fix(web): viewport reactivity, off-screen thumbhashes being rendered

web/package.json

@@ -86,8 +86,7 @@
     "svelte-gestures": "^5.1.3",
     "svelte-i18n": "^4.0.1",
     "svelte-local-storage-store": "^0.6.4",
-    "svelte-maplibre": "^0.9.13",
-    "thumbhash": "^0.1.1"
+    "svelte-maplibre": "^0.9.13"
   },
   "volta": {
     "node": "22.13.1"

web/src/lib/actions/thumbhash.ts

@@ -1,19 +1,137 @@
 import { decodeBase64 } from '$lib/utils';
-import { thumbHashToRGBA } from 'thumbhash';
 
 /**
  * Renders a thumbnail onto a canvas from a base64 encoded hash.
  * @param canvas
  * @param param1 object containing the base64 encoded hash (base64Thumbhash: yourString)
  */
 export function thumbhash(canvas: HTMLCanvasElement, { base64ThumbHash }: { base64ThumbHash: string }) {
-  const ctx = canvas.getContext('2d');
+  const ctx = canvas.getContext('bitmaprenderer');
   if (ctx) {
     const { w, h, rgba } = thumbHashToRGBA(decodeBase64(base64ThumbHash));
-    const pixels = ctx.createImageData(w, h);
-    canvas.width = w;
-    canvas.height = h;
-    pixels.data.set(rgba);
-    ctx.putImageData(pixels, 0, 0);
+    void createImageBitmap(new ImageData(rgba, w, h)).then((bitmap) => ctx.transferFromImageBitmap(bitmap));
   }
 }
+
+// This copyright notice applies to the below code
+// It is a modified version of the original that uses `Uint8ClampedArray` instead of `UInt8Array` and has some trivial typing/linting changes
+
+/* Copyright (c) 2023 Evan Wallace
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.*/
+/**
+ * Decodes a ThumbHash to an RGBA image. RGB is not be premultiplied by A.
+ *
+ * @param hash The bytes of the ThumbHash.
+ * @returns The width, height, and pixels of the rendered placeholder image.
+ */
+export function thumbHashToRGBA(hash: Uint8Array) {
+  const { PI, max, cos, round } = Math;
+
+  // Read the constants
+  const header24 = hash[0] | (hash[1] << 8) | (hash[2] << 16);
+  const header16 = hash[3] | (hash[4] << 8);
+  const l_dc = (header24 & 63) / 63;
+  const p_dc = ((header24 >> 6) & 63) / 31.5 - 1;
+  const q_dc = ((header24 >> 12) & 63) / 31.5 - 1;
+  const l_scale = ((header24 >> 18) & 31) / 31;
+  const hasAlpha = header24 >> 23;
+  const p_scale = ((header16 >> 3) & 63) / 63;
+  const q_scale = ((header16 >> 9) & 63) / 63;
+  const isLandscape = header16 >> 15;
+  const lx = max(3, isLandscape ? (hasAlpha ? 5 : 7) : header16 & 7);
+  const ly = max(3, isLandscape ? header16 & 7 : hasAlpha ? 5 : 7);
+  const a_dc = hasAlpha ? (hash[5] & 15) / 15 : 1;
+  const a_scale = (hash[5] >> 4) / 15;
+
+  // Read the varying factors (boost saturation by 1.25x to compensate for quantization)
+  const ac_start = hasAlpha ? 6 : 5;
+  let ac_index = 0;
+  const decodeChannel = (nx: number, ny: number, scale: number) => {
+    const ac = [];
+    for (let cy = 0; cy < ny; cy++) {
+      for (let cx = cy ? 0 : 1; cx * ny < nx * (ny - cy); cx++) {
+        ac.push((((hash[ac_start + (ac_index >> 1)] >> ((ac_index++ & 1) << 2)) & 15) / 7.5 - 1) * scale);
+      }
+    }
+    return ac;
+  };
+  const l_ac = decodeChannel(lx, ly, l_scale);
+  const p_ac = decodeChannel(3, 3, p_scale * 1.25);
+  const q_ac = decodeChannel(3, 3, q_scale * 1.25);
+  const a_ac = hasAlpha ? decodeChannel(5, 5, a_scale) : null;
+
+  // Decode using the DCT into RGB
+  const ratio = thumbHashToApproximateAspectRatio(hash);
+  const w = round(ratio > 1 ? 32 : 32 * ratio);
+  const h = round(ratio > 1 ? 32 / ratio : 32);
+  const rgba = new Uint8ClampedArray(w * h * 4),
+    fx = [],
+    fy = [];
+  for (let y = 0, i = 0; y < h; y++) {
+    for (let x = 0; x < w; x++, i += 4) {
+      let l = l_dc,
+        p = p_dc,
+        q = q_dc,
+        a = a_dc;
+
+      // Precompute the coefficients
+      for (let cx = 0, n = max(lx, hasAlpha ? 5 : 3); cx < n; cx++) {
+        fx[cx] = cos((PI / w) * (x + 0.5) * cx);
+      }
+      for (let cy = 0, n = max(ly, hasAlpha ? 5 : 3); cy < n; cy++) {
+        fy[cy] = cos((PI / h) * (y + 0.5) * cy);
+      }
+
+      // Decode L
+      for (let cy = 0, j = 0; cy < ly; cy++) {
+        for (let cx = cy ? 0 : 1, fy2 = fy[cy] * 2; cx * ly < lx * (ly - cy); cx++, j++) {
+          l += l_ac[j] * fx[cx] * fy2;
+        }
+      }
+
+      // Decode P and Q
+      for (let cy = 0, j = 0; cy < 3; cy++) {
+        for (let cx = cy ? 0 : 1, fy2 = fy[cy] * 2; cx < 3 - cy; cx++, j++) {
+          const f = fx[cx] * fy2;
+          p += p_ac[j] * f;
+          q += q_ac[j] * f;
+        }
+      }
+
+      // Decode A
+      if (a_ac !== null) {
+        for (let cy = 0, j = 0; cy < 5; cy++) {
+          for (let cx = cy ? 0 : 1, fy2 = fy[cy] * 2; cx < 5 - cy; cx++, j++) {
+            a += a_ac[j] * fx[cx] * fy2;
+          }
+        }
+      }
+
+      // Convert to RGB
+      const b = l - (2 / 3) * p;
+      const r = (3 * l - b + q) / 2;
+      const g = r - q;
+      rgba[i] = 255 * r;
+      rgba[i + 1] = 255 * g;
+      rgba[i + 2] = 255 * b;
+      rgba[i + 3] = 255 * a;
+    }
+  }
+  return { w, h, rgba };
+}
+
+/**
+ * Extracts the approximate aspect ratio of the original image.
+ *
+ * @param hash The bytes of the ThumbHash.
+ * @returns The approximate aspect ratio (i.e. width / height).
+ */
+export function thumbHashToApproximateAspectRatio(hash: Uint8Array) {
+  const header = hash[3];
+  const hasAlpha = hash[2] & 0x80;
+  const isLandscape = hash[4] & 0x80;
+  const lx = isLandscape ? (hasAlpha ? 5 : 7) : header & 7;
+  const ly = isLandscape ? header & 7 : hasAlpha ? 5 : 7;
+  return lx / ly;
+}

web/src/lib/components/assets/thumbnail/thumbnail.svelte

@@ -39,6 +39,7 @@
     thumbnailSize?: number | undefined;
     thumbnailWidth?: number | undefined;
     thumbnailHeight?: number | undefined;
+    eagerThumbhash?: boolean;
     selected?: boolean;
     selectionCandidate?: boolean;
     disabled?: boolean;
@@ -71,6 +72,7 @@
     thumbnailSize = undefined,
     thumbnailWidth = undefined,
     thumbnailHeight = undefined,
+    eagerThumbhash = true,
     selected = false,
     selectionCandidate = false,
     disabled = false,
@@ -113,7 +115,6 @@
 
   let width = $derived(thumbnailSize || thumbnailWidth || 235);
   let height = $derived(thumbnailSize || thumbnailHeight || 235);
-  let display = $derived(intersecting);
 
   const onIconClickedHandler = (e?: MouseEvent) => {
     e?.stopPropagation();
@@ -207,7 +208,11 @@
     ? 'bg-gray-300'
     : 'bg-immich-primary/20 dark:bg-immich-dark-primary/20'}"
 >
-  {#if !loaded && asset.thumbhash}
+  <!-- TODO: Rendering thumbhashes for offscreen assets is a ton of overhead.
+             This is here to ensure thumbhashes appear on the first 
+             frame instead of a gray box for smaller date groups,
+             where the overhead (while wasteful) does not cause major issues. -->
+  {#if eagerThumbhash && !loaded && asset.thumbhash}
     <canvas
       use:thumbhash={{ base64ThumbHash: asset.thumbhash }}
       class="absolute object-cover z-10"
@@ -217,7 +222,17 @@
     ></canvas>
   {/if}
 
-  {#if display}
+  {#if intersecting}
+    {#if !eagerThumbhash && !loaded && asset.thumbhash}
+      <canvas
+        use:thumbhash={{ base64ThumbHash: asset.thumbhash }}
+        class="absolute object-cover z-10"
+        style:width="{width}px"
+        style:height="{height}px"
+        out:fade={{ duration: THUMBHASH_FADE_DURATION }}
+      ></canvas>
+    {/if}
+
     <!-- svelte queries for all links on afterNavigate, leading to performance problems in asset-grid which updates
      the navigation url on scroll. Replace this with button for now. -->
     <div

web/src/lib/components/photos-page/asset-date-group.svelte

@@ -38,7 +38,7 @@
   $: dateGroups = bucket.dateGroups;
 
   const {
-    DATEGROUP: { INTERSECTION_ROOT_TOP, INTERSECTION_ROOT_BOTTOM },
+    DATEGROUP: { INTERSECTION_ROOT_TOP, INTERSECTION_ROOT_BOTTOM, SMALL_GROUP_THRESHOLD },
   } = TUNABLES;
   /* TODO figure out a way to calculate this*/
   const TITLE_HEIGHT = 51;
@@ -179,6 +179,7 @@
           >
             {#each dateGroup.assets as asset, index (asset.id)}
               {@const box = dateGroup.geometry.boxes[index]}
+              {@const isSmallGroup = dateGroup.assets.length <= SMALL_GROUP_THRESHOLD}
               <!-- update ASSET_GRID_PADDING-->
               <div
                 use:intersectionObserver={{
@@ -217,6 +218,7 @@
                   disabled={$assetStore.albumAssets.has(asset.id)}
                   thumbnailWidth={box.width}
                   thumbnailHeight={box.height}
+                  eagerThumbhash={isSmallGroup}
                 />
               </div>
             {/each}

web/src/lib/components/photos-page/asset-grid.svelte

@@ -20,7 +20,7 @@
   } from '$lib/utils/timeline-util';
   import { TUNABLES } from '$lib/utils/tunables';
   import type { AlbumResponseDto, AssetResponseDto, PersonResponseDto } from '@immich/sdk';
-  import { throttle } from 'lodash-es';
+  import { debounce, throttle } from 'lodash-es';
   import { onDestroy, onMount, type Snippet } from 'svelte';
   import Portal from '../shared-components/portal/portal.svelte';
   import Scrubber from '../shared-components/scrubber/scrubber.svelte';
@@ -81,8 +81,9 @@
 
   let { isViewing: showAssetViewer, asset: viewingAsset, preloadAssets, gridScrollTarget } = assetViewingStore;
 
-  const viewport: ViewportXY = $state({ width: 0, height: 0, x: 0, y: 0 });
-  const safeViewport: ViewportXY = $state({ width: 0, height: 0, x: 0, y: 0 });
+  // this does *not* need to be reactive and making it reactive causes expensive repeated updates
+  // svelte-ignore non_reactive_update
+  let safeViewport: ViewportXY = { width: 0, height: 0, x: 0, y: 0 };
 
   const componentId = generateId();
   let element: HTMLElement | undefined = $state();
@@ -103,7 +104,7 @@
   let leadout = $state(false);
 
   const {
-    ASSET_GRID: { NAVIGATE_ON_ASSET_IN_VIEW },
+    ASSET_GRID: { NAVIGATE_ON_ASSET_IN_VIEW, LARGE_BUCKET_THRESHOLD, LARGE_BUCKET_DEBOUNCE_MS },
     BUCKET: {
       INTERSECTION_ROOT_TOP: BUCKET_INTERSECTION_ROOT_TOP,
       INTERSECTION_ROOT_BOTTOM: BUCKET_INTERSECTION_ROOT_BOTTOM,
@@ -114,14 +115,6 @@
     },
   } = TUNABLES;
 
-  const isViewportOrigin = () => {
-    return viewport.height === 0 && viewport.width === 0;
-  };
-
-  const isEqual = (a: ViewportXY, b: ViewportXY) => {
-    return a.height == b.height && a.width == b.width && a.x === b.x && a.y === b.y;
-  };
-
   const completeNav = () => {
     navigating = false;
     if (internalScroll) {
@@ -235,6 +228,14 @@
   };
 
   onMount(() => {
+    if (element) {
+      const rect = element.getBoundingClientRect();
+      safeViewport.height = rect.height;
+      safeViewport.width = rect.width;
+      safeViewport.x = rect.x;
+      safeViewport.y = rect.y;
+    }
+
     void $assetStore
       .init({ bucketListener })
       .then(() => ($assetStore.connect(), $assetStore.updateViewport(safeViewport)));
@@ -259,8 +260,6 @@
     }
     return offset;
   }
-  const _updateViewport = () => void $assetStore.updateViewport(safeViewport);
-  const updateViewport = throttle(_updateViewport, 16);
 
   const getMaxScrollPercent = () =>
     ($assetStore.timelineHeight + bottomSectionHeight + topSectionHeight - safeViewport.height) /
@@ -744,23 +743,8 @@
     }
   });
 
-  $effect(() => {
-    if (element && isViewportOrigin()) {
-      const rect = element.getBoundingClientRect();
-      viewport.height = rect.height;
-      viewport.width = rect.width;
-      viewport.x = rect.x;
-      viewport.y = rect.y;
-    }
-    if (!isViewportOrigin() && !isEqual(viewport, safeViewport)) {
-      safeViewport.height = viewport.height;
-      safeViewport.width = viewport.width;
-      safeViewport.x = viewport.x;
-      safeViewport.y = viewport.y;
-      updateViewport();
-    }
-  });
-
+  let largeBucketMode = false;
+  let updateViewport = debounce(() => $assetStore.updateViewport(safeViewport), 8);
   let shortcutList = $derived(
     (() => {
       if ($isSearchEnabled || $showAssetViewer) {
@@ -843,7 +827,21 @@
   id="asset-grid"
   class="scrollbar-hidden h-full overflow-y-auto outline-none {isEmpty ? 'm-0' : 'ml-4 tall:ml-0 mr-[60px]'}"
   tabindex="-1"
-  use:resizeObserver={({ height, width }) => ((viewport.width = width), (viewport.height = height))}
+  use:resizeObserver={({ width, height }) => {
+    if (!largeBucketMode && assetStore.maxBucketAssets >= LARGE_BUCKET_THRESHOLD) {
+      largeBucketMode = true;
+      // Each viewport update causes each asset to re-decode both the thumbhash and the thumbnail.
+      // This is because the thumbnail components are destroyed and re-mounted, possibly because of the intersection observer.
+      // For larger buckets, this can lead to freezing and a poor user experience.
+      // As a mitigation, we aggressively debounce the viewport update to reduce the number of these events.
+      updateViewport = debounce(() => $assetStore.updateViewport(safeViewport), LARGE_BUCKET_DEBOUNCE_MS, {
+        leading: false,
+        trailing: true,
+      });
+    }
+    safeViewport = { width, height, x: safeViewport.x, y: safeViewport.y };
+    void updateViewport();
+  }}
   bind:this={element}
   onscroll={() => ((assetStore.lastScrollTime = Date.now()), handleTimelineScroll())}
 >

web/src/lib/stores/assets.store.ts

@@ -232,6 +232,7 @@ export class AssetStore {
   albumAssets: Set<string> = new Set();
   pendingScrollBucket: AssetBucket | undefined;
   pendingScrollAssetId: string | undefined;
+  maxBucketAssets = 0;
 
   listeners: BucketListener[] = [];
 
@@ -560,6 +561,7 @@ export class AssetStore {
 
       bucket.assets = assets;
       bucket.dateGroups = splitBucketIntoDateGroups(bucket, get(locale));
+      this.maxBucketAssets = Math.max(this.maxBucketAssets, assets.length);
       this.updateGeometry(bucket, true);
       this.timelineHeight = this.buckets.reduce((accumulator, b) => accumulator + b.bucketHeight, 0);
       bucket.loaded();