feat: preload DNSLink websites in background

With
#826
data was no longer getting into the cache of local node.
This adds background queue that ensures every visited DNSLink asset
is preloaded to the cache of local IPFS node.

Author: lidel

add-on/_locales/en/messages.json

@@ -287,6 +287,14 @@
     "message": "If global redirect is enabled, this will include DNSLink websites and redirect them to respective /ipns/{fqdn} paths at Custom Gateway",
     "description": "An option description on the Preferences screen (option_dnslinkRedirect_description)"
   },
+  "option_dnslinkDataPreload_title": {
+    "message": "Preload visited pages",
+    "description": "An option title on the Preferences screen (option_dnslinkDataPreload_title)"
+  },
+  "option_dnslinkDataPreload_description": {
+    "message": "DNSLink websites will be preloaded to the local IPFS node to enable offline access and improve resiliency against network failures",
+    "description": "An option description on the Preferences screen (option_dnslinkDataPreload_description)"
+  },
   "option_dnslinkRedirect_warning": {
     "message": "Redirecting to a path-based gateway breaks Origin-based security isolation of DNSLink website! Please leave this disabled unless you are aware of (and ok with) related risks.",
     "description": "A warning on the Preferences screen, displayed when URL does not belong to Secure Context (option_customGatewayUrl_warning)"

add-on/src/lib/dnslink.js

@@ -17,8 +17,11 @@ module.exports = function createDnslinkResolver (getState) {
   // DNSLink lookup result cache
   const cacheOptions = { max: 1000, maxAge: 1000 * 60 * 60 * 12 }
   const cache = new LRU(cacheOptions)
-  // upper bound for concurrent background lookups done by preloadDnslink(url)
-  const lookupQueue = new PQueue({ concurrency: 8 })
+  // upper bound for concurrent background lookups done by resolve(url)
+  const lookupQueue = new PQueue({ concurrency: 4 })
+  // preload of DNSLink data
+  const preloadUrlCache = new LRU(cacheOptions)
+  const preloadQueue = new PQueue({ concurrency: 4 })
 
   const dnslinkResolver = {
 
@@ -89,20 +92,34 @@ module.exports = function createDnslinkResolver (getState) {
       return dnslink
     },
 
-    // does not return anything, runs async lookup in the background
-    // and saves result into cache with an optional callback
-    preloadDnslink (url, cb) {
-      if (dnslinkResolver.canLookupURL(url)) {
-        lookupQueue.add(async () => {
-          const fqdn = new URL(url).hostname
-          const result = dnslinkResolver.readAndCacheDnslink(fqdn)
-          if (cb) {
-            cb(result)
-          }
-        })
-      } else if (cb) {
-        cb(null)
-      }
+    // runs async lookup in a queue in the background and returns the record
+    async resolve (url) {
+      if (!dnslinkResolver.canLookupURL(url)) return
+      const fqdn = new URL(url).hostname
+      const cachedResult = dnslinkResolver.cachedDnslink(fqdn)
+      if (cachedResult) return cachedResult
+      return lookupQueue.add(() => {
+        return dnslinkResolver.readAndCacheDnslink(fqdn)
+      })
+    },
+
+    // preloads data behind the url to local node
+    async preloadData (url) {
+      const state = getState()
+      if (!state.dnslinkDataPreload) return
+      if (preloadUrlCache.get(url)) return
+      preloadUrlCache.set(url, true)
+      const dnslink = await dnslinkResolver.resolve(url)
+      if (!dnslink) return
+      if (state.ipfsNodeType === 'embedded') return
+      if (state.peerCount < 1) return
+      return preloadQueue.add(async () => {
+        const { pathname } = new URL(url)
+        const preloadUrl = new URL(state.gwURLString)
+        preloadUrl.pathname = `${dnslink}${pathname}`
+        await fetch(preloadUrl.toString(), { method: 'HEAD' })
+        return preloadUrl
+      })
     },
 
     // low level lookup without cache

add-on/src/lib/ipfs-request.js

@@ -89,7 +89,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     if (request.type === 'main_frame') {
       // lazily trigger DNSLink lookup (will do anything only if status for root domain is not in cache)
       if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
-        dnslinkResolver.preloadDnslink(request.url)
+        dnslinkResolver.resolve(request.url) // no await: preload record in background
       }
     }
     return isIgnored(request.requestId)
@@ -142,15 +142,18 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
           return redirectToGateway(request.url, state, ipfsPathValidator)
         }
         // Detect dnslink using heuristics enabled in Preferences
-        if (state.dnslinkRedirect && state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
-          const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url)
-          if (dnslinkRedirect && isSafeToRedirect(request, runtime)) {
-            // console.log('onBeforeRequest.dnslinkRedirect', dnslinkRedirect)
-            return dnslinkRedirect
+        if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
+          if (state.dnslinkRedirect) {
+            const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url)
+            if (dnslinkRedirect && isSafeToRedirect(request, runtime)) {
+              // console.log('onBeforeRequest.dnslinkRedirect', dnslinkRedirect)
+              return dnslinkRedirect
+            }
+          } else if (state.dnslinkDataPreload) {
+            dnslinkResolver.preloadData(request.url)
           }
           if (state.dnslinkPolicy === 'best-effort') {
-            // dnslinkResolver.preloadDnslink(request.url, (dnslink) => console.log(`---> preloadDnslink(${new URL(request.url).hostname})=${dnslink}`))
-            dnslinkResolver.preloadDnslink(request.url)
+            dnslinkResolver.resolve(request.url)
           }
         }
       }

add-on/src/lib/options.js

@@ -17,6 +17,7 @@ exports.optionDefaults = Object.freeze({
   automaticMode: true,
   linkify: false,
   dnslinkPolicy: 'best-effort',
+  dnslinkDataPreload: true,
   dnslinkRedirect: false,
   recoverFailedHttpRequests: true,
   detectIpfsPathHeader: true,

add-on/src/options/forms/dnslink-form.js

@@ -7,11 +7,13 @@ const switchToggle = require('../../pages/components/switch-toggle')
 
 function dnslinkForm ({
   dnslinkPolicy,
+  dnslinkDataPreload,
   dnslinkRedirect,
   onOptionChange
 }) {
   const onDnslinkPolicyChange = onOptionChange('dnslinkPolicy')
   const onDnslinkRedirectChange = onOptionChange('dnslinkRedirect')
+  const onDnslinkDataPreloadChange = onOptionChange('dnslinkDataPreload')
 
   return html`
     <form>
@@ -47,6 +49,15 @@ function dnslinkForm ({
             </option>
           </select>
         </div>
+        <div>
+          <label for="dnslinkDataPreload">
+            <dl>
+              <dt>${browser.i18n.getMessage('option_dnslinkDataPreload_title')}</dt>
+              <dd>${browser.i18n.getMessage('option_dnslinkDataPreload_description')}</dd>
+            </dl>
+          </label>
+          <div>${switchToggle({ id: 'dnslinkDataPreload', checked: dnslinkDataPreload, onchange: onDnslinkDataPreloadChange })}</div>
+        </div>
         <div>
           <label for="dnslinkRedirect">
             <dl>

add-on/src/options/page.js

@@ -80,6 +80,7 @@ module.exports = function optionsPage (state, emit) {
   })}
   ${dnslinkForm({
     dnslinkPolicy: state.options.dnslinkPolicy,
+    dnslinkDataPreload: state.options.dnslinkDataPreload,
     dnslinkRedirect: state.options.dnslinkRedirect,
     onOptionChange
   })}