From 9c7aaf7790b74f3a4e94ef383e9da5cdaa03dd44 Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Sat, 19 Oct 2019 00:00:06 -0400 Subject: [PATCH 01/12] add subdomainPublicGatewayUrl option --- add-on/_locales/en/messages.json | 8 ++++++++ add-on/src/lib/ipfs-companion.js | 4 ++++ add-on/src/lib/options.js | 1 + add-on/src/lib/state.js | 3 +++ add-on/src/options/forms/gateways-form.js | 20 ++++++++++++++++++++ add-on/src/options/page.js | 1 + add-on/src/popup/browser-action/store.js | 1 + 7 files changed, 38 insertions(+) diff --git a/add-on/_locales/en/messages.json b/add-on/_locales/en/messages.json index 85c7b310d..5e647e065 100644 --- a/add-on/_locales/en/messages.json +++ b/add-on/_locales/en/messages.json @@ -291,6 +291,14 @@ "message": "Fallback URL used when Custom Gateway is not available and for copying shareable links", "description": "An option description on the Preferences screen (option_publicGatewayUrl_description)" }, + "option_subdomainPublicGatewayUrl_title": { + "message": "Default Subdomain Public Gateway", + "description": "An option title on the Preferences screen (option_subdomainPublicGatewayUrl_title)" + }, + "option_subdomainPublicGatewayUrl_description": { + "message": "Default subdomain public gateway for recovery of broken subdomain gateways", + "description": "An option description on the Preferences screen (option_subdomainPublicGatewayUrl_description)" + }, "option_header_api": { "message": "API", "description": "A section header on the Preferences screen (option_header_api)" diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js index 82fb05fc3..da9aea65b 100644 --- a/add-on/src/lib/ipfs-companion.js +++ b/add-on/src/lib/ipfs-companion.js @@ -685,6 +685,10 @@ module.exports = async function init () { state.pubGwURL = new URL(change.newValue) state.pubGwURLString = state.pubGwURL.toString() break + case 'subdomainPublicGatewayUrl': + state.subdomainGwURL = new URL(change.newValue) + state.subdomainGwURLString = state.subdomainGwURL.toString() + break case 'useCustomGateway': state.redirect = change.newValue break diff --git a/add-on/src/lib/options.js b/add-on/src/lib/options.js index cc43ef8b4..74f035749 100644 --- a/add-on/src/lib/options.js +++ b/add-on/src/lib/options.js @@ -11,6 +11,7 @@ exports.optionDefaults = Object.freeze({ ipfsNodeType: buildDefaultIpfsNodeType(), ipfsNodeConfig: buildDefaultIpfsNodeConfig(), publicGatewayUrl: 'https://ipfs.io', + subdomainPublicGatewayUrl: 'https://ipfs.dweb.link', useCustomGateway: true, noRedirectHostnames: [], automaticMode: true, diff --git a/add-on/src/lib/state.js b/add-on/src/lib/state.js index a948d60ab..7cc02c533 100644 --- a/add-on/src/lib/state.js +++ b/add-on/src/lib/state.js @@ -17,6 +17,9 @@ function initState (options) { state.pubGwURL = safeURL(options.publicGatewayUrl) state.pubGwURLString = state.pubGwURL.toString() delete state.publicGatewayUrl + state.subdomainGwURL = safeURL(options.subdomainPublicGatewayUrl) + state.subdomainGwURLString = state.subdomainGwURL.toString() + delete state.subdomainPublicGatewayUrl state.redirect = options.useCustomGateway delete state.useCustomGateway state.apiURL = safeURL(options.ipfsApiUrl) diff --git a/add-on/src/options/forms/gateways-form.js b/add-on/src/options/forms/gateways-form.js index 6eec690d9..47146c125 100644 --- a/add-on/src/options/forms/gateways-form.js +++ b/add-on/src/options/forms/gateways-form.js @@ -16,11 +16,13 @@ function gatewaysForm ({ useCustomGateway, noRedirectHostnames, publicGatewayUrl, + subdomainPublicGatewayUrl, onOptionChange }) { const onCustomGatewayUrlChange = onOptionChange('customGatewayUrl', normalizeGatewayURL) const onUseCustomGatewayChange = onOptionChange('useCustomGateway') const onPublicGatewayUrlChange = onOptionChange('publicGatewayUrl', normalizeGatewayURL) + const onSubdomainPublicGatewayUrlChange = onOptionChange('subdomainPublicGatewayUrl', normalizeGatewayURL) const onNoRedirectHostnamesChange = onOptionChange('noRedirectHostnames', hostTextToArray) const mixedContentWarning = !secureContextUrl.test(customGatewayUrl) const supportRedirectToCustomGateway = ipfsNodeType !== 'embedded' @@ -99,6 +101,24 @@ function gatewaysForm ({ >${hostArrayToText(noRedirectHostnames)} ` : null} +
+ + +
` diff --git a/add-on/src/options/page.js b/add-on/src/options/page.js index 5e3637b95..3d31888ea 100644 --- a/add-on/src/options/page.js +++ b/add-on/src/options/page.js @@ -66,6 +66,7 @@ module.exports = function optionsPage (state, emit) { customGatewayUrl: state.options.customGatewayUrl, useCustomGateway: state.options.useCustomGateway, publicGatewayUrl: state.options.publicGatewayUrl, + subdomainPublicGatewayUrl: state.options.subdomainPublicGatewayUrl, noRedirectHostnames: state.options.noRedirectHostnames, onOptionChange })} diff --git a/add-on/src/popup/browser-action/store.js b/add-on/src/popup/browser-action/store.js index 586120d4e..3c0035a6e 100644 --- a/add-on/src/popup/browser-action/store.js +++ b/add-on/src/popup/browser-action/store.js @@ -23,6 +23,7 @@ module.exports = (state, emitter) => { isIpfsOnline: false, ipfsApiUrl: null, publicGatewayUrl: null, + subdomainPublicGatewayUrl: null, gatewayAddress: null, swarmPeers: null, gatewayVersion: null, From 15c292416494b41f7cfd9fc3791c89d1430596a8 Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Sun, 20 Oct 2019 18:05:08 -0400 Subject: [PATCH 02/12] add broken subdomain recovery --- add-on/src/lib/ipfs-path.js | 28 +++++++++++++++++++++++++++- add-on/src/lib/ipfs-request.js | 23 +++++++++++++++-------- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/add-on/src/lib/ipfs-path.js b/add-on/src/lib/ipfs-path.js index 4ff23ce21..ef011edb8 100644 --- a/add-on/src/lib/ipfs-path.js +++ b/add-on/src/lib/ipfs-path.js @@ -38,6 +38,17 @@ function pathAtHttpGateway (path, gatewayUrl) { } exports.pathAtHttpGateway = pathAtHttpGateway +function redirectSubdomainGateway (url, subdomainGateway) { + if (typeof url === 'string') { + url = new URL(url) + } + const fqdn = url.hostname.split('.') + const cid = fqdn[0] + const protocol = fqdn[1] + return trimDoubleSlashes(`${subdomainGateway.protocol}//${cid}.${protocol}.${subdomainGateway.hostname}${url.pathname}${url.search}${url.hash}`) +} +exports.redirectSubdomainGateway = redirectSubdomainGateway + function trimDoubleSlashes (urlString) { return urlString.replace(/([^:]\/)\/+/g, '$1') } @@ -72,7 +83,11 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { validIpfsOrIpnsPath (path) { return validIpfsOrIpnsPath(path, dnslinkResolver) }, - + // Test if URL is a subdomain gateway resource + // TODO: add test if URL is a public subdomain resource + ipfsOrIpnsSubdomain (url) { + return IsIpfs.subdomain(url) + }, // Test if actions such as 'copy URL', 'pin/unpin' should be enabled for the URL isIpfsPageActionsContext (url) { return Boolean(url && !url.startsWith(getState().apiURLString) && ( @@ -108,6 +123,17 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { // Return original URL (eg. DNSLink domains) or null if not an URL return input.startsWith('http') ? input : null }, + // Resolve URL or path to subdomain gateway + // - non-subdomain path is returned as-is + // The purpose of this resolver is to return a valid IPFS + // subdomain path + resolveToSubdomainUrl (url, gatewayUrl) { + // if non-subdomain return as-is + if (!IsIpfs.subdomain(url)) return false + + const gateway = gatewayUrl || getState().subdomainGwURL + return redirectSubdomainGateway(url, gateway) + }, // Resolve URL or path to IPFS Path: // - The path can be /ipfs/ or /ipns/ diff --git a/add-on/src/lib/ipfs-request.js b/add-on/src/lib/ipfs-request.js index 60e73cb58..edcb29b9d 100644 --- a/add-on/src/lib/ipfs-request.js +++ b/add-on/src/lib/ipfs-request.js @@ -392,7 +392,12 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru // Check if error can be recovered by opening same content-addresed path // using active gateway (public or local, depending on redirect state) if (isRecoverable(request, state, ipfsPathValidator)) { - const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) + let redirectUrl + if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) { + redirectUrl = ipfsPathValidator.resolveToSubdomainUrl(request.url, state.subdomainGwURL) + } else { + redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) + } log(`onErrorOccurred: attempting to recover from network error (${request.error}) for ${request.url}`, redirectUrl) return createTabWithURL({ redirectUrl }, browser) } @@ -404,13 +409,15 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru const state = getState() if (!state.active) return if (request.statusCode === 200) return // finish if no error to recover from + let redirectUrl if (isRecoverable(request, state, ipfsPathValidator)) { - const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) - const redirect = { redirectUrl } - if (redirect) { - log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirect) - return createTabWithURL(redirect, browser) + if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) { + redirectUrl = ipfsPathValidator.resolveToSubdomainUrl(request.url, state.subdomainGwURL) + } else { + redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) } + log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirectUrl) + return createTabWithURL({ redirectUrl: 'https://wikipedia.org' }, browser) } } } @@ -527,8 +534,8 @@ function isRecoverable (request, state, ipfsPathValidator) { return state.recoverFailedHttpRequests && request.type === 'main_frame' && (recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) && - ipfsPathValidator.publicIpfsOrIpnsResource(request.url) && - !request.url.startsWith(state.pubGwURLString) + (ipfsPathValidator.publicIpfsOrIpnsResource(request.url) || ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) && + !request.url.startsWith(state.pubGwURLString) && !request.url.includes(state.subdomainGwURL.hostname) } // Recovery check for onErrorOccurred (request.error) From 344954b7d4ee2eb90673f9643661df4e9b4f8909 Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Sun, 20 Oct 2019 18:05:08 -0400 Subject: [PATCH 03/12] add broken subdomain recovery --- add-on/src/lib/ipfs-path.js | 28 +++++++++++++++++++++++++++- add-on/src/lib/ipfs-request.js | 23 +++++++++++++++-------- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/add-on/src/lib/ipfs-path.js b/add-on/src/lib/ipfs-path.js index 4ff23ce21..ef011edb8 100644 --- a/add-on/src/lib/ipfs-path.js +++ b/add-on/src/lib/ipfs-path.js @@ -38,6 +38,17 @@ function pathAtHttpGateway (path, gatewayUrl) { } exports.pathAtHttpGateway = pathAtHttpGateway +function redirectSubdomainGateway (url, subdomainGateway) { + if (typeof url === 'string') { + url = new URL(url) + } + const fqdn = url.hostname.split('.') + const cid = fqdn[0] + const protocol = fqdn[1] + return trimDoubleSlashes(`${subdomainGateway.protocol}//${cid}.${protocol}.${subdomainGateway.hostname}${url.pathname}${url.search}${url.hash}`) +} +exports.redirectSubdomainGateway = redirectSubdomainGateway + function trimDoubleSlashes (urlString) { return urlString.replace(/([^:]\/)\/+/g, '$1') } @@ -72,7 +83,11 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { validIpfsOrIpnsPath (path) { return validIpfsOrIpnsPath(path, dnslinkResolver) }, - + // Test if URL is a subdomain gateway resource + // TODO: add test if URL is a public subdomain resource + ipfsOrIpnsSubdomain (url) { + return IsIpfs.subdomain(url) + }, // Test if actions such as 'copy URL', 'pin/unpin' should be enabled for the URL isIpfsPageActionsContext (url) { return Boolean(url && !url.startsWith(getState().apiURLString) && ( @@ -108,6 +123,17 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { // Return original URL (eg. DNSLink domains) or null if not an URL return input.startsWith('http') ? input : null }, + // Resolve URL or path to subdomain gateway + // - non-subdomain path is returned as-is + // The purpose of this resolver is to return a valid IPFS + // subdomain path + resolveToSubdomainUrl (url, gatewayUrl) { + // if non-subdomain return as-is + if (!IsIpfs.subdomain(url)) return false + + const gateway = gatewayUrl || getState().subdomainGwURL + return redirectSubdomainGateway(url, gateway) + }, // Resolve URL or path to IPFS Path: // - The path can be /ipfs/ or /ipns/ diff --git a/add-on/src/lib/ipfs-request.js b/add-on/src/lib/ipfs-request.js index 60e73cb58..9bb64f7d2 100644 --- a/add-on/src/lib/ipfs-request.js +++ b/add-on/src/lib/ipfs-request.js @@ -392,7 +392,12 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru // Check if error can be recovered by opening same content-addresed path // using active gateway (public or local, depending on redirect state) if (isRecoverable(request, state, ipfsPathValidator)) { - const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) + let redirectUrl + if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) { + redirectUrl = ipfsPathValidator.resolveToSubdomainUrl(request.url, state.subdomainGwURL) + } else { + redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) + } log(`onErrorOccurred: attempting to recover from network error (${request.error}) for ${request.url}`, redirectUrl) return createTabWithURL({ redirectUrl }, browser) } @@ -404,13 +409,15 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru const state = getState() if (!state.active) return if (request.statusCode === 200) return // finish if no error to recover from + let redirectUrl if (isRecoverable(request, state, ipfsPathValidator)) { - const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) - const redirect = { redirectUrl } - if (redirect) { - log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirect) - return createTabWithURL(redirect, browser) + if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) { + redirectUrl = ipfsPathValidator.resolveToSubdomainUrl(request.url, state.subdomainGwURL) + } else { + redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) } + log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirectUrl) + return createTabWithURL({ redirectUrl }, browser) } } } @@ -527,8 +534,8 @@ function isRecoverable (request, state, ipfsPathValidator) { return state.recoverFailedHttpRequests && request.type === 'main_frame' && (recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) && - ipfsPathValidator.publicIpfsOrIpnsResource(request.url) && - !request.url.startsWith(state.pubGwURLString) + (ipfsPathValidator.publicIpfsOrIpnsResource(request.url) || ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) && + !request.url.startsWith(state.pubGwURLString) && !request.url.includes(state.subdomainGwURL.hostname) } // Recovery check for onErrorOccurred (request.error) From 936414e6130d7cd424a9d57d6ed7ca5a34dfc284 Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Sun, 20 Oct 2019 23:03:15 -0400 Subject: [PATCH 04/12] add tests for subdomain recovery --- .../lib/ipfs-request-gateway-recover.test.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/functional/lib/ipfs-request-gateway-recover.test.js b/test/functional/lib/ipfs-request-gateway-recover.test.js index 8f1ce6438..529deafdf 100644 --- a/test/functional/lib/ipfs-request-gateway-recover.test.js +++ b/test/functional/lib/ipfs-request-gateway-recover.test.js @@ -46,6 +46,16 @@ describe('requestHandler.onCompleted:', function () { // HTTP-level errors state.recoverFailedHttpRequests = true state.dnslinkPolicy = false }) + it('should do nothing if broken request is default subdomain', async function () { + const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.dweb.link/wiki/', 500) + await requestHandler.onCompleted(request) + assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') + }) + it('should redirect to default subdomain gateway on broken subdomain gateway request', async function () { + const request = urlRequestWithStatus('http://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.brokenexample.com/wiki/', 500) + await requestHandler.onCompleted(request) + assert.ok(browser.tabs.create.withArgs({ url: 'https://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.dweb.link/wiki/', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with default subdomain gateway URL') + }) it('should do nothing if broken request is a non-IPFS request', async function () { const request = urlRequestWithStatus('https://wikipedia.org', 500) await requestHandler.onCompleted(request) @@ -78,6 +88,11 @@ describe('requestHandler.onCompleted:', function () { // HTTP-level errors state.recoverFailedHttpRequests = false state.dnslinkPolicy = false }) + it('should do nothing on failed subdomain gateway request', async function () { + const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.brokendomain.com/wiki/', 500) + await requestHandler.onCompleted(request) + assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') + }) it('should do nothing on broken non-default public gateway IPFS request', async function () { const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500) await requestHandler.onCompleted(request) @@ -120,6 +135,16 @@ describe('requestHandler.onErrorOccurred:', function () { // network errors state.recoverFailedHttpRequests = true state.dnslinkPolicy = false }) + it('should do nothing if failed request is default subdomain', async function () { + const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.dweb.link/wiki/', 500) + await requestHandler.onErrorOccurred(request) + assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') + }) + it('should redirect to default subdomain gateway on failed subdomain gateway request', async function () { + const request = urlRequestWithStatus('http://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.brokenexample.com/wiki/', 500) + await requestHandler.onErrorOccurred(request) + assert.ok(browser.tabs.create.withArgs({ url: 'https://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.dweb.link/wiki/', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with default subdomain gateway URL') + }) it('should do nothing if failed request is a non-IPFS request', async function () { const request = urlRequestWithNetworkError('https://wikipedia.org') await requestHandler.onErrorOccurred(request) @@ -178,6 +203,11 @@ describe('requestHandler.onErrorOccurred:', function () { // network errors await requestHandler.onErrorOccurred(request) assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') }) + it('should do nothing on failed subdomain gateway request', async function () { + const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.brokendomain.com/wiki/', 500) + await requestHandler.onErrorOccurred(request) + assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') + }) it('should do nothing on unreachable HTTP server with DNSLink', async function () { state.dnslinkPolicy = 'best-effort' dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco') From 289be35172cc77890cffa77ac87d2e39c1528e1c Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Tue, 22 Oct 2019 20:40:15 -0400 Subject: [PATCH 05/12] return url from resolveToSubdomainUrl if not valid subdomain --- add-on/src/lib/ipfs-path.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/add-on/src/lib/ipfs-path.js b/add-on/src/lib/ipfs-path.js index ef011edb8..ac0b6a9eb 100644 --- a/add-on/src/lib/ipfs-path.js +++ b/add-on/src/lib/ipfs-path.js @@ -129,7 +129,7 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { // subdomain path resolveToSubdomainUrl (url, gatewayUrl) { // if non-subdomain return as-is - if (!IsIpfs.subdomain(url)) return false + if (!IsIpfs.subdomain(url)) return url const gateway = gatewayUrl || getState().subdomainGwURL return redirectSubdomainGateway(url, gateway) From 7b2c6525466ddf22cff0ad3ec215f8b4fcf84bbb Mon Sep 17 00:00:00 2001 From: Colin Fruit <17092461+colinfruit@users.noreply.github.com> Date: Wed, 23 Oct 2019 20:02:52 -0400 Subject: [PATCH 06/12] Update test/functional/lib/ipfs-request-gateway-recover.test.js Co-Authored-By: Marcin Rataj --- test/functional/lib/ipfs-request-gateway-recover.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/lib/ipfs-request-gateway-recover.test.js b/test/functional/lib/ipfs-request-gateway-recover.test.js index 529deafdf..15acb7a24 100644 --- a/test/functional/lib/ipfs-request-gateway-recover.test.js +++ b/test/functional/lib/ipfs-request-gateway-recover.test.js @@ -46,7 +46,7 @@ describe('requestHandler.onCompleted:', function () { // HTTP-level errors state.recoverFailedHttpRequests = true state.dnslinkPolicy = false }) - it('should do nothing if broken request is default subdomain', async function () { + it('should do nothing if broken request is for the default subdomain gateway', async function () { const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.dweb.link/wiki/', 500) await requestHandler.onCompleted(request) assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') From 612daa540e139e93127d7699b6346e074a380c2e Mon Sep 17 00:00:00 2001 From: Colin Fruit <17092461+colinfruit@users.noreply.github.com> Date: Wed, 23 Oct 2019 20:03:07 -0400 Subject: [PATCH 07/12] Update test/functional/lib/ipfs-request-gateway-recover.test.js Co-Authored-By: Marcin Rataj --- test/functional/lib/ipfs-request-gateway-recover.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/lib/ipfs-request-gateway-recover.test.js b/test/functional/lib/ipfs-request-gateway-recover.test.js index 15acb7a24..9a409ab89 100644 --- a/test/functional/lib/ipfs-request-gateway-recover.test.js +++ b/test/functional/lib/ipfs-request-gateway-recover.test.js @@ -135,7 +135,7 @@ describe('requestHandler.onErrorOccurred:', function () { // network errors state.recoverFailedHttpRequests = true state.dnslinkPolicy = false }) - it('should do nothing if failed request is default subdomain', async function () { + it('should do nothing if failed request is for the default subdomain gateway', async function () { const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.dweb.link/wiki/', 500) await requestHandler.onErrorOccurred(request) assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called') From 3d4a9c05d96217d445833b211396163fbebf27b1 Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Wed, 23 Oct 2019 20:32:56 -0400 Subject: [PATCH 08/12] update gateway-form --- add-on/src/options/forms/gateways-form.js | 41 +++++++++++++---------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/add-on/src/options/forms/gateways-form.js b/add-on/src/options/forms/gateways-form.js index 47146c125..b381d2428 100644 --- a/add-on/src/options/forms/gateways-form.js +++ b/add-on/src/options/forms/gateways-form.js @@ -50,6 +50,29 @@ function gatewaysForm ({ onchange=${onPublicGatewayUrlChange} value=${publicGatewayUrl} /> +
+ + +
${supportRedirectToCustomGateway && allowChangeOfCustomGateway ? html`
` : null} -
- - -
` From e44c5f0f070512391bbc6493c38241e20d5b91fd Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Wed, 23 Oct 2019 20:48:16 -0400 Subject: [PATCH 09/12] update subdomain variable naming --- add-on/_locales/en/messages.json | 8 ++++---- add-on/src/lib/ipfs-companion.js | 6 +++--- add-on/src/lib/ipfs-path.js | 4 ++-- add-on/src/lib/ipfs-request.js | 10 +++++----- add-on/src/lib/options.js | 2 +- add-on/src/lib/state.js | 6 +++--- add-on/src/options/forms/gateways-form.js | 16 ++++++++-------- add-on/src/options/page.js | 2 +- add-on/src/popup/browser-action/store.js | 2 +- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/add-on/_locales/en/messages.json b/add-on/_locales/en/messages.json index 5e647e065..ffe916402 100644 --- a/add-on/_locales/en/messages.json +++ b/add-on/_locales/en/messages.json @@ -291,13 +291,13 @@ "message": "Fallback URL used when Custom Gateway is not available and for copying shareable links", "description": "An option description on the Preferences screen (option_publicGatewayUrl_description)" }, - "option_subdomainPublicGatewayUrl_title": { + "option_publicSubdomainGatewayUrl_title": { "message": "Default Subdomain Public Gateway", - "description": "An option title on the Preferences screen (option_subdomainPublicGatewayUrl_title)" + "description": "An option title on the Preferences screen (option_publicSubdomainGatewayUrl_title)" }, - "option_subdomainPublicGatewayUrl_description": { + "option_publicSubdomainGatewayUrl_description": { "message": "Default subdomain public gateway for recovery of broken subdomain gateways", - "description": "An option description on the Preferences screen (option_subdomainPublicGatewayUrl_description)" + "description": "An option description on the Preferences screen (option_publicSubdomainGatewayUrl_description)" }, "option_header_api": { "message": "API", diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js index da9aea65b..caca4dd96 100644 --- a/add-on/src/lib/ipfs-companion.js +++ b/add-on/src/lib/ipfs-companion.js @@ -685,9 +685,9 @@ module.exports = async function init () { state.pubGwURL = new URL(change.newValue) state.pubGwURLString = state.pubGwURL.toString() break - case 'subdomainPublicGatewayUrl': - state.subdomainGwURL = new URL(change.newValue) - state.subdomainGwURLString = state.subdomainGwURL.toString() + case 'publicSubdomainGatewayUrl': + state.pubSubdomainGwURL = new URL(change.newValue) + state.pubSubdomainGwURLString = state.pubSubdomainGwURL.toString() break case 'useCustomGateway': state.redirect = change.newValue diff --git a/add-on/src/lib/ipfs-path.js b/add-on/src/lib/ipfs-path.js index ac0b6a9eb..f3a8709d4 100644 --- a/add-on/src/lib/ipfs-path.js +++ b/add-on/src/lib/ipfs-path.js @@ -127,11 +127,11 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { // - non-subdomain path is returned as-is // The purpose of this resolver is to return a valid IPFS // subdomain path - resolveToSubdomainUrl (url, gatewayUrl) { + resolveToPublicSubdomainUrl (url, gatewayUrl) { // if non-subdomain return as-is if (!IsIpfs.subdomain(url)) return url - const gateway = gatewayUrl || getState().subdomainGwURL + const gateway = gatewayUrl || getState().pubSubdomainGwURL return redirectSubdomainGateway(url, gateway) }, diff --git a/add-on/src/lib/ipfs-request.js b/add-on/src/lib/ipfs-request.js index 123da4694..1ed26a3ef 100644 --- a/add-on/src/lib/ipfs-request.js +++ b/add-on/src/lib/ipfs-request.js @@ -393,9 +393,9 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru // using active gateway (public or local, depending on redirect state) if (isRecoverable(request, state, ipfsPathValidator)) { let redirectUrl - // if subdomain request redirect to default subdomain url + // if subdomain request redirect to default public subdomain url if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) { - redirectUrl = ipfsPathValidator.resolveToSubdomainUrl(request.url, state.subdomainGwURL) + redirectUrl = ipfsPathValidator.resolveToPublicSubdomainUrl(request.url, state.pubSubdomainGwURL) } else { redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) } @@ -412,9 +412,9 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru if (request.statusCode === 200) return // finish if no error to recover from let redirectUrl if (isRecoverable(request, state, ipfsPathValidator)) { - // if subdomain request redirect to default subdomain url + // if subdomain request redirect to default public subdomain url if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) { - redirectUrl = ipfsPathValidator.resolveToSubdomainUrl(request.url, state.subdomainGwURL) + redirectUrl = ipfsPathValidator.resolveToPublicSubdomainUrl(request.url, state.pubSubdomainGwURL) } else { redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString) } @@ -537,7 +537,7 @@ function isRecoverable (request, state, ipfsPathValidator) { request.type === 'main_frame' && (recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) && (ipfsPathValidator.publicIpfsOrIpnsResource(request.url) || ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) && - !request.url.startsWith(state.pubGwURLString) && !request.url.includes(state.subdomainGwURL.hostname) + !request.url.startsWith(state.pubGwURLString) && !request.url.includes(state.pubSubdomainGwURL.hostname) } // Recovery check for onErrorOccurred (request.error) diff --git a/add-on/src/lib/options.js b/add-on/src/lib/options.js index e033bf6ca..9ed986a1f 100644 --- a/add-on/src/lib/options.js +++ b/add-on/src/lib/options.js @@ -11,7 +11,7 @@ exports.optionDefaults = Object.freeze({ ipfsNodeType: buildDefaultIpfsNodeType(), ipfsNodeConfig: buildDefaultIpfsNodeConfig(), publicGatewayUrl: 'https://ipfs.io', - subdomainPublicGatewayUrl: 'https://dweb.link', + publicSubdomainGatewayUrl: 'https://dweb.link', useCustomGateway: true, noRedirectHostnames: [], automaticMode: true, diff --git a/add-on/src/lib/state.js b/add-on/src/lib/state.js index 7cc02c533..8d493c076 100644 --- a/add-on/src/lib/state.js +++ b/add-on/src/lib/state.js @@ -17,9 +17,9 @@ function initState (options) { state.pubGwURL = safeURL(options.publicGatewayUrl) state.pubGwURLString = state.pubGwURL.toString() delete state.publicGatewayUrl - state.subdomainGwURL = safeURL(options.subdomainPublicGatewayUrl) - state.subdomainGwURLString = state.subdomainGwURL.toString() - delete state.subdomainPublicGatewayUrl + state.pubSubdomainGwURL = safeURL(options.publicSubdomainGatewayUrl) + state.pubSubdomainGwURLString = state.pubSubdomainGwURL.toString() + delete state.publicSubdomainGatewayUrl state.redirect = options.useCustomGateway delete state.useCustomGateway state.apiURL = safeURL(options.ipfsApiUrl) diff --git a/add-on/src/options/forms/gateways-form.js b/add-on/src/options/forms/gateways-form.js index b381d2428..09e9f3d9d 100644 --- a/add-on/src/options/forms/gateways-form.js +++ b/add-on/src/options/forms/gateways-form.js @@ -16,13 +16,13 @@ function gatewaysForm ({ useCustomGateway, noRedirectHostnames, publicGatewayUrl, - subdomainPublicGatewayUrl, + publicSubdomainGatewayUrl, onOptionChange }) { const onCustomGatewayUrlChange = onOptionChange('customGatewayUrl', normalizeGatewayURL) const onUseCustomGatewayChange = onOptionChange('useCustomGateway') const onPublicGatewayUrlChange = onOptionChange('publicGatewayUrl', normalizeGatewayURL) - const onSubdomainPublicGatewayUrlChange = onOptionChange('subdomainPublicGatewayUrl', normalizeGatewayURL) + const onPublicSubdomainGatewayUrlChange = onOptionChange('publicSubdomainGatewayUrl', normalizeGatewayURL) const onNoRedirectHostnamesChange = onOptionChange('noRedirectHostnames', hostTextToArray) const mixedContentWarning = !secureContextUrl.test(customGatewayUrl) const supportRedirectToCustomGateway = ipfsNodeType !== 'embedded' @@ -51,11 +51,11 @@ function gatewaysForm ({ value=${publicGatewayUrl} />
-
${supportRedirectToCustomGateway && allowChangeOfCustomGateway ? html`
diff --git a/add-on/src/options/page.js b/add-on/src/options/page.js index 3d31888ea..942162f55 100644 --- a/add-on/src/options/page.js +++ b/add-on/src/options/page.js @@ -66,7 +66,7 @@ module.exports = function optionsPage (state, emit) { customGatewayUrl: state.options.customGatewayUrl, useCustomGateway: state.options.useCustomGateway, publicGatewayUrl: state.options.publicGatewayUrl, - subdomainPublicGatewayUrl: state.options.subdomainPublicGatewayUrl, + publicSubdomainGatewayUrl: state.options.publicSubdomainGatewayUrl, noRedirectHostnames: state.options.noRedirectHostnames, onOptionChange })} diff --git a/add-on/src/popup/browser-action/store.js b/add-on/src/popup/browser-action/store.js index 3c0035a6e..64a748273 100644 --- a/add-on/src/popup/browser-action/store.js +++ b/add-on/src/popup/browser-action/store.js @@ -23,7 +23,7 @@ module.exports = (state, emitter) => { isIpfsOnline: false, ipfsApiUrl: null, publicGatewayUrl: null, - subdomainPublicGatewayUrl: null, + publicSubdomainGatewayUrl: null, gatewayAddress: null, swarmPeers: null, gatewayVersion: null, From cbc71e5d997f9d372b8a09021509e9986faf5a8e Mon Sep 17 00:00:00 2001 From: colinfruit <17092461+colinfruit@users.noreply.github.com> Date: Wed, 23 Oct 2019 21:41:04 -0400 Subject: [PATCH 10/12] use subdomainPattern to extract protocol and cid from subdomain urls --- add-on/src/lib/ipfs-path.js | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/add-on/src/lib/ipfs-path.js b/add-on/src/lib/ipfs-path.js index f3a8709d4..7aa623758 100644 --- a/add-on/src/lib/ipfs-path.js +++ b/add-on/src/lib/ipfs-path.js @@ -24,11 +24,13 @@ function subdomainToIpfsPath (url) { if (typeof url === 'string') { url = new URL(url) } - const fqdn = url.hostname.split('.') + const match = url.toString().match(IsIpfs.subdomainPattern) + if (!match) throw new Error('no match for IsIpfs.subdomainPattern') + // TODO: support CID split with commas - const cid = fqdn[0] + const cid = match[1] // TODO: support .ip(f|n)s. being at deeper levels - const protocol = fqdn[1] + const protocol = match[2] return `/${protocol}/${cid}${url.pathname}${url.search}${url.hash}` } @@ -42,9 +44,10 @@ function redirectSubdomainGateway (url, subdomainGateway) { if (typeof url === 'string') { url = new URL(url) } - const fqdn = url.hostname.split('.') - const cid = fqdn[0] - const protocol = fqdn[1] + const match = url.toString().match(IsIpfs.subdomainPattern) + if (!match) throw new Error('no match for IsIpfs.subdomainPattern') + const cid = match[1] + const protocol = match[2] return trimDoubleSlashes(`${subdomainGateway.protocol}//${cid}.${protocol}.${subdomainGateway.hostname}${url.pathname}${url.search}${url.hash}`) } exports.redirectSubdomainGateway = redirectSubdomainGateway @@ -126,12 +129,12 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) { // Resolve URL or path to subdomain gateway // - non-subdomain path is returned as-is // The purpose of this resolver is to return a valid IPFS - // subdomain path - resolveToPublicSubdomainUrl (url, gatewayUrl) { + // subdomain URL + resolveToPublicSubdomainUrl (url, optionalGatewayUrl) { // if non-subdomain return as-is if (!IsIpfs.subdomain(url)) return url - const gateway = gatewayUrl || getState().pubSubdomainGwURL + const gateway = optionalGatewayUrl || getState().pubSubdomainGwURL return redirectSubdomainGateway(url, gateway) }, From 4ed8d11b8c2901eb2e0b4a5a2440545b4b138ad3 Mon Sep 17 00:00:00 2001 From: Colin Fruit <17092461+colinfruit@users.noreply.github.com> Date: Thu, 24 Oct 2019 20:15:40 -0400 Subject: [PATCH 11/12] Update add-on/_locales/en/messages.json Co-Authored-By: Marcin Rataj --- add-on/_locales/en/messages.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/add-on/_locales/en/messages.json b/add-on/_locales/en/messages.json index ffe916402..637a11d1e 100644 --- a/add-on/_locales/en/messages.json +++ b/add-on/_locales/en/messages.json @@ -292,7 +292,7 @@ "description": "An option description on the Preferences screen (option_publicGatewayUrl_description)" }, "option_publicSubdomainGatewayUrl_title": { - "message": "Default Subdomain Public Gateway", + "message": "Default Public Subdomain Gateway", "description": "An option title on the Preferences screen (option_publicSubdomainGatewayUrl_title)" }, "option_publicSubdomainGatewayUrl_description": { From 8f8199c5deeb11c8cbe0723f8758d30c19490de0 Mon Sep 17 00:00:00 2001 From: Colin Fruit <17092461+colinfruit@users.noreply.github.com> Date: Thu, 24 Oct 2019 20:15:54 -0400 Subject: [PATCH 12/12] Update add-on/_locales/en/messages.json Co-Authored-By: Marcin Rataj --- add-on/_locales/en/messages.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/add-on/_locales/en/messages.json b/add-on/_locales/en/messages.json index 637a11d1e..841ed34f6 100644 --- a/add-on/_locales/en/messages.json +++ b/add-on/_locales/en/messages.json @@ -296,7 +296,7 @@ "description": "An option title on the Preferences screen (option_publicSubdomainGatewayUrl_title)" }, "option_publicSubdomainGatewayUrl_description": { - "message": "Default subdomain public gateway for recovery of broken subdomain gateways", + "message": "Default public subdomain gateway for recovery of broken subdomain gateways", "description": "An option description on the Preferences screen (option_publicSubdomainGatewayUrl_description)" }, "option_header_api": {