Skip to content

Commit 614da95

Browse files
colinfruitlidel
authored andcommitted
feat: recover from failed HTTP requests to third party gateways (#783)
This closes #640, enabling dead public gateways to be redirected to the default public gateway. This includes DNSLink websites that went offline. Added as an experimental option enabled by default.
1 parent 8512691 commit 614da95

File tree

7 files changed

+262
-13
lines changed

7 files changed

+262
-13
lines changed

add-on/_locales/en/messages.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,14 @@
375375
"message": "Check before HTTP request",
376376
"description": "A select field option description on the Preferences screen (option_dnslinkPolicy_enabled)"
377377
},
378+
"option_recoverFailedHttpRequests_title": {
379+
"message": "Recover Failed HTTP Requests",
380+
"description": "An option title on the Preferences screen (option_recoverFailedHttpRequests_title)"
381+
},
382+
"option_recoverFailedHttpRequests_description": {
383+
"message": "Recover failed HTTP requests for IPFS resources by redirecting to the public gateway.",
384+
"description": "An option description on the Preferences screen (option_recoverFailedHttpRequests_description)"
385+
},
378386
"option_detectIpfsPathHeader_title": {
379387
"message": "Detect X-Ipfs-Path Header",
380388
"description": "An option title on the Preferences screen (option_detectIpfsPathHeader_title)"

add-on/src/lib/ipfs-companion.js

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ module.exports = async function init () {
106106
browser.webRequest.onBeforeRequest.addListener(onBeforeRequest, { urls: ['<all_urls>'] }, ['blocking'])
107107
browser.webRequest.onHeadersReceived.addListener(onHeadersReceived, { urls: ['<all_urls>'] }, ['blocking', 'responseHeaders'])
108108
browser.webRequest.onErrorOccurred.addListener(onErrorOccurred, { urls: ['<all_urls>'] })
109+
browser.webRequest.onCompleted.addListener(onCompleted, { urls: ['<all_urls>'] })
109110
browser.storage.onChanged.addListener(onStorageChange)
110111
browser.webNavigation.onCommitted.addListener(onNavigationCommitted)
111112
browser.webNavigation.onDOMContentLoaded.addListener(onDOMContentLoaded)
@@ -170,6 +171,10 @@ module.exports = async function init () {
170171
return modifyRequest.onErrorOccurred(request)
171172
}
172173

174+
function onCompleted (request) {
175+
return modifyRequest.onCompleted(request)
176+
}
177+
173178
// RUNTIME MESSAGES (one-off messaging)
174179
// ===================================================================
175180
// https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/runtime/sendMessage
@@ -693,6 +698,9 @@ module.exports = async function init () {
693698
await browser.storage.local.set({ detectIpfsPathHeader: true })
694699
}
695700
break
701+
case 'recoverFailedHttpRequests':
702+
state[key] = change.newValue
703+
break
696704
case 'logNamespaces':
697705
shouldReloadExtension = true
698706
state[key] = localStorage.debug = change.newValue

add-on/src/lib/ipfs-request.js

Lines changed: 70 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,26 @@ const recoverableErrors = new Set([
2020
'net::ERR_INTERNET_DISCONNECTED' // no network
2121
])
2222

23+
const recoverableErrorCodes = new Set([
24+
404,
25+
408,
26+
410,
27+
415,
28+
451,
29+
500,
30+
502,
31+
503,
32+
504,
33+
509,
34+
520,
35+
521,
36+
522,
37+
523,
38+
524,
39+
525,
40+
526
41+
])
42+
2343
// Request modifier provides event listeners for the various stages of making an HTTP request
2444
// API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
2545
function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, runtime) {
@@ -380,29 +400,48 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
380400
// console.log('onErrorOccurred:' + request.error)
381401
// console.log('onErrorOccurred', request)
382402
// Check if error is final and can be recovered via DNSLink
403+
let redirect
383404
const recoverableViaDnslink =
384405
state.dnslinkPolicy &&
385406
request.type === 'main_frame' &&
386407
recoverableErrors.has(request.error)
387408
if (recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)) {
388409
// Explicit call to ignore global DNSLink policy and force DNS TXT lookup
389410
const cachedDnslink = dnslinkResolver.readAndCacheDnslink(new URL(request.url).hostname)
390-
const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
391-
// We can't redirect in onErrorOccurred, so if DNSLink is present
392-
// recover by opening IPNS version in a new tab
393-
// TODO: add tests and demo
394-
if (dnslinkRedirect) {
395-
log(`onErrorOccurred: recovering using dnslink for ${request.url}`, dnslinkRedirect)
396-
const currentTabId = await browser.tabs.query({ active: true, currentWindow: true }).then(tabs => tabs[0].id)
397-
await browser.tabs.create({
398-
active: true,
399-
openerTabId: currentTabId,
400-
url: dnslinkRedirect.redirectUrl
401-
})
411+
redirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
412+
log(`onErrorOccurred: attempting to recover using dnslink for ${request.url}`, redirect)
413+
}
414+
// if error cannot be recovered via DNSLink
415+
// direct the request to the public gateway
416+
const recoverable = isRecoverable(request, state, ipfsPathValidator)
417+
if (!redirect && recoverable) {
418+
const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
419+
redirect = { redirectUrl }
420+
log(`onErrorOccurred: attempting to recover failed request for ${request.url}`, redirect)
421+
}
422+
// We can't redirect in onErrorOccurred, so if DNSLink is present
423+
// recover by opening IPNS version in a new tab
424+
// TODO: add tests and demo
425+
if (redirect) {
426+
createTabWithURL(redirect, browser)
427+
}
428+
},
429+
430+
async onCompleted (request) {
431+
const state = getState()
432+
433+
const recoverable =
434+
isRecoverable(request, state, ipfsPathValidator) &&
435+
recoverableErrorCodes.has(request.statusCode)
436+
if (recoverable) {
437+
const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
438+
const redirect = { redirectUrl }
439+
if (redirect) {
440+
log(`onCompleted: attempting to recover failed request for ${request.url}`, redirect)
441+
createTabWithURL(redirect, browser)
402442
}
403443
}
404444
}
405-
406445
}
407446
}
408447

@@ -508,3 +547,21 @@ function normalizedUnhandledIpfsProtocol (request, pubGwUrl) {
508547
function findHeaderIndex (name, headers) {
509548
return headers.findIndex(x => x.name && x.name.toLowerCase() === name.toLowerCase())
510549
}
550+
551+
// utility functions for handling redirects
552+
// from onErrorOccurred and onCompleted
553+
function isRecoverable (request, state, ipfsPathValidator) {
554+
return state.recoverFailedHttpRequests &&
555+
ipfsPathValidator.publicIpfsOrIpnsResource(request.url) &&
556+
!request.url.startsWith(state.pubGwURLString) &&
557+
request.type === 'main_frame'
558+
}
559+
560+
async function createTabWithURL (redirect, browser) {
561+
const currentTabId = await browser.tabs.query({ active: true, currentWindow: true }).then(tabs => tabs[0].id)
562+
await browser.tabs.create({
563+
active: true,
564+
openerTabId: currentTabId,
565+
url: redirect.redirectUrl
566+
})
567+
}

add-on/src/lib/options.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ exports.optionDefaults = Object.freeze({
1616
automaticMode: true,
1717
linkify: false,
1818
dnslinkPolicy: 'best-effort',
19+
recoverFailedHttpRequests: true,
1920
detectIpfsPathHeader: true,
2021
preloadAtPublicGateway: true,
2122
catchUnhandledProtocols: true,

add-on/src/options/forms/experiments-form.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ function experimentsForm ({
1111
catchUnhandledProtocols,
1212
linkify,
1313
dnslinkPolicy,
14+
recoverFailedHttpRequests,
1415
detectIpfsPathHeader,
1516
ipfsProxy,
1617
logNamespaces,
@@ -22,6 +23,7 @@ function experimentsForm ({
2223
const onCatchUnhandledProtocolsChange = onOptionChange('catchUnhandledProtocols')
2324
const onLinkifyChange = onOptionChange('linkify')
2425
const onDnslinkPolicyChange = onOptionChange('dnslinkPolicy')
26+
const onrecoverFailedHttpRequestsChange = onOptionChange('recoverFailedHttpRequests')
2527
const onDetectIpfsPathHeaderChange = onOptionChange('detectIpfsPathHeader')
2628
const onIpfsProxyChange = onOptionChange('ipfsProxy')
2729

@@ -96,6 +98,15 @@ function experimentsForm ({
9698
</option>
9799
</select>
98100
</div>
101+
<div>
102+
<label for="recoverFailedHttpRequests">
103+
<dl>
104+
<dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
105+
<dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
106+
</dl>
107+
</label>
108+
<div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
109+
</div>
99110
<div>
100111
<label for="detectIpfsPathHeader">
101112
<dl>

add-on/src/options/page.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ module.exports = function optionsPage (state, emit) {
7575
catchUnhandledProtocols: state.options.catchUnhandledProtocols,
7676
linkify: state.options.linkify,
7777
dnslinkPolicy: state.options.dnslinkPolicy,
78+
recoverFailedHttpRequests: state.options.recoverFailedHttpRequests,
7879
detectIpfsPathHeader: state.options.detectIpfsPathHeader,
7980
ipfsProxy: state.options.ipfsProxy,
8081
logNamespaces: state.options.logNamespaces,
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
'use strict'
2+
const { describe, it, before, beforeEach, after, afterEach } = require('mocha')
3+
const sinon = require('sinon')
4+
const { assert } = require('chai')
5+
const { URL } = require('url')
6+
const browser = require('sinon-chrome')
7+
const { initState } = require('../../../add-on/src/lib/state')
8+
const { createRuntimeChecks } = require('../../../add-on/src/lib/runtime-checks')
9+
const { createRequestModifier } = require('../../../add-on/src/lib/ipfs-request')
10+
const createDnslinkResolver = require('../../../add-on/src/lib/dnslink')
11+
const { createIpfsPathValidator } = require('../../../add-on/src/lib/ipfs-path')
12+
const { optionDefaults } = require('../../../add-on/src/lib/options')
13+
14+
const url2request = (url, type = 'main_frame') => {
15+
return { url, type }
16+
}
17+
const urlRequestWithStatus = (url, statusCode = 200, type = 'main_frame') => {
18+
return { ...url2request(url, type), statusCode }
19+
}
20+
21+
describe('requestHandler.onCompleted:', function () {
22+
let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
23+
24+
before(function () {
25+
global.URL = URL
26+
browser.tabs = { ...browser.tabs, query: sinon.stub().resolves([{ id: 20 }]) }
27+
global.browser = browser
28+
})
29+
30+
beforeEach(async function () {
31+
state = initState(optionDefaults)
32+
const getState = () => state
33+
const getIpfs = () => {}
34+
dnslinkResolver = createDnslinkResolver(getState)
35+
runtime = Object.assign({}, await createRuntimeChecks(browser)) // make it mutable for tests
36+
ipfsPathValidator = createIpfsPathValidator(getState, getIpfs, dnslinkResolver)
37+
requestHandler = createRequestModifier(getState, dnslinkResolver, ipfsPathValidator, runtime)
38+
})
39+
40+
describe('with recoverFailedHttpRequests=true', function () {
41+
beforeEach(function () {
42+
state.recoverFailedHttpRequests = true
43+
})
44+
it('should do nothing if broken request is a non-IPFS request', async function () {
45+
const request = urlRequestWithStatus('https://wikipedia.org', 500)
46+
await requestHandler.onCompleted(request)
47+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
48+
})
49+
it('should do nothing if broken request is a non-public IPFS request', async function () {
50+
const request = urlRequestWithStatus('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
51+
await requestHandler.onCompleted(request)
52+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
53+
})
54+
it('should do nothing if broken request is to the default public gateway', async function () {
55+
const request = urlRequestWithStatus('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
56+
await requestHandler.onCompleted(request)
57+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
58+
})
59+
it('should do nothing if broken request is not a \'main_frame\' request', async function () {
60+
const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500, 'stylesheet')
61+
await requestHandler.onCompleted(request)
62+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
63+
})
64+
it('should redirect broken non-default public gateway IPFS request to public gateway', async function () {
65+
const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
66+
await requestHandler.onCompleted(request)
67+
assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
68+
})
69+
})
70+
71+
describe('with recoverFailedHttpRequests=false', function () {
72+
beforeEach(function () {
73+
state.recoverFailedHttpRequests = false
74+
})
75+
it('should do nothing on broken non-default public gateway IPFS request', async function () {
76+
const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
77+
await requestHandler.onCompleted(request)
78+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
79+
})
80+
})
81+
82+
afterEach(function () {
83+
browser.tabs.create.reset()
84+
})
85+
86+
after(function () {
87+
delete global.url
88+
delete global.browser
89+
browser.flush()
90+
})
91+
})
92+
93+
describe('requestHandler.onErrorOccurred:', function () {
94+
let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
95+
96+
before(function () {
97+
global.URL = URL
98+
browser.tabs = { ...browser.tabs, query: sinon.stub().resolves([{ id: 20 }]) }
99+
global.browser = browser
100+
})
101+
102+
beforeEach(async function () {
103+
state = initState(optionDefaults)
104+
const getState = () => state
105+
const getIpfs = () => {}
106+
dnslinkResolver = createDnslinkResolver(getState)
107+
runtime = Object.assign({}, await createRuntimeChecks(browser)) // make it mutable for tests
108+
ipfsPathValidator = createIpfsPathValidator(getState, getIpfs, dnslinkResolver)
109+
requestHandler = createRequestModifier(getState, dnslinkResolver, ipfsPathValidator, runtime)
110+
})
111+
112+
describe('with recoverFailedHttpRequests=true', function () {
113+
beforeEach(function () {
114+
state.recoverFailedHttpRequests = true
115+
})
116+
it('should do nothing if failed request is a non-IPFS request', async function () {
117+
const request = url2request('https://wikipedia.org', 500)
118+
await requestHandler.onErrorOccurred(request)
119+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
120+
})
121+
it('should do nothing if failed request is a non-public IPFS request', async function () {
122+
const request = url2request('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
123+
await requestHandler.onErrorOccurred(request)
124+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
125+
})
126+
it('should do nothing if failed request is to the default public gateway', async function () {
127+
const request = url2request('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
128+
await requestHandler.onErrorOccurred(request)
129+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
130+
})
131+
it('should do nothing if failed request is not a \'main_frame\' request', async function () {
132+
const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'stylesheet')
133+
await requestHandler.onErrorOccurred(request)
134+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
135+
})
136+
it('should redirect failed non-default public gateway IPFS request to public gateway', async function () {
137+
const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
138+
await requestHandler.onErrorOccurred(request)
139+
assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
140+
})
141+
})
142+
143+
describe('with recoverFailedHttpRequests=false', function () {
144+
beforeEach(function () {
145+
state.recoverFailedHttpRequests = false
146+
})
147+
it('should do nothing on failed non-default public gateway IPFS request', async function () {
148+
const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
149+
await requestHandler.onErrorOccurred(request)
150+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
151+
})
152+
})
153+
154+
afterEach(function () {
155+
browser.tabs.create.reset()
156+
})
157+
158+
after(function () {
159+
delete global.url
160+
delete global.browser
161+
browser.flush()
162+
})
163+
})

0 commit comments

Comments
 (0)