Skip to content

feat: support provider query parameter #242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: support provider query parameter #242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c30629a
feat: support provider query parameter
vasco-santos May 23, 2025
03a573d
fix: add allow provider parameter option to have provider query param…
vasco-santos May 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/verified-fetch/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@
"@libp2p/webrtc": "^5.2.14",
"@libp2p/websockets": "^9.2.12",
"@multiformats/dns": "^1.0.6",
"@multiformats/multiaddr": "^12.4.0",
"cborg": "^4.2.11",
"file-type": "^20.5.0",
"helia": "^5.4.1",
Expand Down
12 changes: 12 additions & 0 deletions packages/verified-fetch/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1034,6 +1034,18 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledP
*/
allowInsecure?: boolean

/**
* By default we will not parse provider query parameters, and will not
* connect to any hosts over their multiaddresses. Instead, we will use the
* default discovery mechanism to find providers for the content.
* This is an experimental feature, and may become the default in the future.
* If you pass `true` here, we will parse the `provider` query parameter and
* connect to the provider specified in the query parameter to retrieve the content.
*
* @default false
*/
allowProviderParameter?: boolean

/**
* Whether to include server-timing headers in the response for an individual request.
*
Expand Down
5 changes: 3 additions & 2 deletions packages/verified-fetch/src/plugins/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type { ContentTypeParser, RequestFormatShorthand } from '../types.js'
import type { ByteRangeContext } from '../utils/byte-range-context.js'
import type { ParsedUrlStringResults } from '../utils/parse-url-string.js'
import type { PathWalkerResponse } from '../utils/walk-path.js'
import type { ProviderOptions } from '@helia/interface'
import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface'
import type { Helia } from 'helia'
import type { Blockstore } from 'interface-blockstore'
Expand All @@ -18,7 +19,7 @@ import type { CustomProgressEvent } from 'progress-events'
*/
export interface PluginOptions {
logger: ComponentLogger
getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions): Blockstore
getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions & ProviderOptions): Blockstore
handleServerTiming<T>(name: string, description: string, fn: () => Promise<T>, withServerTiming: boolean): Promise<T>
contentTypeParser?: ContentTypeParser
helia: Helia
Expand All @@ -42,7 +43,7 @@ export interface PluginContext extends ParsedUrlStringResults {
modified: number
withServerTiming?: boolean
onProgress?(evt: CustomProgressEvent<any>): void
options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions & ProviderOptions
isDirectory?: boolean
directoryEntries?: UnixFSEntry[]
errors?: PluginError[]
Expand Down
3 changes: 2 additions & 1 deletion packages/verified-fetch/src/utils/parse-resource.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ export async function parseResource (resource: Resource, { ipns, logger }: Parse
query: {},
ipfsPath: `/ipfs/${cid.toString()}`,
ttl: 29030400, // 1 year for ipfs content
serverTimings: []
serverTimings: [],
providers: []
} satisfies ParsedUrlStringResults
}

Expand Down
26 changes: 25 additions & 1 deletion packages/verified-fetch/src/utils/parse-url-string.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { AbortError } from '@libp2p/interface'
import { multiaddr } from '@multiformats/multiaddr'
import { CID } from 'multiformats/cid'
import { getPeerIdFromString } from './get-peer-id-from-string.js'
import { serverTiming } from './server-timing.js'
Expand All @@ -7,6 +8,7 @@ import type { ServerTimingResult } from './server-timing.js'
import type { RequestFormatShorthand } from '../types.js'
import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface'
import type { Multiaddr } from '@multiformats/multiaddr'
import type { ProgressOptions } from 'progress-events'

const ipnsCache = new TLRU<DNSLinkResolveResult | IPNSResolveResult>(1000)
Expand Down Expand Up @@ -50,6 +52,11 @@ export interface ParsedUrlStringResults extends ResolveResult {
* serverTiming items
*/
serverTimings: Array<ServerTimingResult<any>>

/**
* The providers hinted in the URL.
*/
providers: Array<Multiaddr>
}

const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
Expand Down Expand Up @@ -286,12 +293,28 @@ export async function parseUrlString ({ urlString, ipns, logger, withServerTimin

// parse query string
const query: Record<string, any> = {}
const providers: Array<Multiaddr> = []

if (queryString != null && queryString.length > 0) {
const queryParts = queryString.split('&')
for (const part of queryParts) {
const [key, value] = part.split('=')
query[key] = decodeURIComponent(value)
// see https://github.com/vasco-santos/provider-hinted-uri
// provider is a special case, the parameter MAY be repeated
if (key === 'provider') {
Comment on lines +302 to +304
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this will impact verified-fetch and inbrowser.link, are you planning to open IPIP about ?provider in https://github.com/ipfs/specs/pulls?

I would softly block on having a spec PR that we can comment on :)

FYI the usual path towards extending Gateway interface (implemented by verfified-fetch) is to follow light IPIP process:

  1. Open PR with IPIP memo in https://github.com/ipfs/specs/ that describes rationale for new feature:
  2. Create implementation PR in GO (https://github.com/ipfs/boxo/tree/main/gateway used by Rainbow and Kubo/Desktop) and JS (Helia/verified-fetch) to ensure the spec is clear enough to be implemented in two different places
  3. Merge the spec once no concerns were raised and after both implementations shipped and work the same.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For visibility, we discussed out of bound and agreed on a first stage:

  1. Make provider query parameter in helia be behind a flag
  2. Draft PR with IPIP to specs with query parameter (no protocol encoding for now)
  3. Enable inbrowser.link to be able to use this on its configuration

Hopefully, we can then iterate a next step after user validation and feedback for:

  1. Add this to Kubo/Go side
  2. Make it the default + Finish/Merge the IPIP
  3. (Eventually) extend this with protocol encoding in multiaddr

Copy link
Member Author

@vasco-santos vasco-santos May 26, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is the IPIP draft to the specs repo ipfs/specs#504

if (query[key] == null) {
query[key] = []
}
const decodedValue = decodeURIComponent(value)
try {
// Must be a multiaddr to be used as Hint
const m = multiaddr(decodedValue)
providers.push(m)
;(query[key] as string[]).push(decodedValue)
} catch {}
} else {
query[key] = decodeURIComponent(value)
}
}

if (query.download != null) {
Expand All @@ -310,6 +333,7 @@ export async function parseUrlString ({ urlString, ipns, logger, withServerTimin
query,
ttl,
ipfsPath: `/${protocol}/${cidOrPeerIdOrDnsLink}${urlPath != null && urlPath !== '' ? `/${urlPath}` : ''}`,
providers,
serverTimings
} satisfies ParsedUrlStringResults
}
Expand Down
9 changes: 6 additions & 3 deletions packages/verified-fetch/src/verified-fetch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { serverTiming } from './utils/server-timing.js'
import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, ResourceDetail, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
import type { VerifiedFetchPlugin, PluginContext, PluginOptions } from './plugins/types.js'
import type { ParsedUrlStringResults } from './utils/parse-url-string.js'
import type { Helia, SessionBlockstore } from '@helia/interface'
import type { Helia, SessionBlockstore, ProviderOptions } from '@helia/interface'
import type { IPNS } from '@helia/ipns'
import type { AbortOptions, Logger } from '@libp2p/interface'
import type { Blockstore } from 'interface-blockstore'
Expand Down Expand Up @@ -121,7 +121,7 @@ export class VerifiedFetch {
this.log.trace('created VerifiedFetch instance')
}

private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions = {}): Blockstore {
private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions & ProviderOptions = {}): Blockstore {
const key = resourceToSessionCacheKey(resource)
if (!useSession) {
return this.helia.blockstore
Expand Down Expand Up @@ -374,7 +374,10 @@ export class VerifiedFetch {
...parsedResult,
resource: resource.toString(),
accept,
options,
options: {
...options,
providers: options?.allowProviderParameter ? parsedResult.providers : undefined
},
withServerTiming,
onProgress: options?.onProgress,
modified: 0
Expand Down
162 changes: 162 additions & 0 deletions packages/verified-fetch/test/utils/parse-url-string.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,60 @@ describe('parseUrlString', () => {
}
)
})

it('can parse URL with CID+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '',
query: {
format: 'tar',
provider: providers
}
}
)
})

it('can parse URL with CID+path+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
query: {
format: 'tar',
provider: providers
}
}
)
})

it('can parse URL with CID+queryString where query string has providers, but one is not valid', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'not-a-multiaddr'
]
await assertMatchUrl(
`ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '',
query: {
format: 'tar',
provider: [providers[0]]
}
}
)
})
})

describe('ipns://<dnsLinkDomain> URLs', () => {
Expand Down Expand Up @@ -357,6 +411,42 @@ describe('parseUrlString', () => {
}
)
})

it('can parse an IPFS path with CID+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '',
query: {
format: 'tar',
provider: providers
}
}
)
})

it('can parse an IPFS path with CID+path+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
query: {
format: 'tar',
provider: providers
}
}
)
})
})

describe('http://example.com/ipfs/<CID> URLs', () => {
Expand Down Expand Up @@ -407,6 +497,42 @@ describe('parseUrlString', () => {
}
)
})

it('can parse an IPFS Gateway URL with CID+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`http://example.com/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '',
query: {
format: 'tar',
provider: providers
}
}
)
})

it('can parse an IPFS Gateway URL with CID+path+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`http://example.com/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
query: {
format: 'tar',
provider: providers
}
}
)
})
})

describe('http://<CID>.ipfs.example.com URLs', () => {
Expand Down Expand Up @@ -457,6 +583,42 @@ describe('parseUrlString', () => {
}
)
})

it('can parse an IPFS Subdomain Gateway URL with CID+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`http://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm.ipfs.example.com?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '',
query: {
format: 'tar',
provider: providers
}
}
)
})

it('can parse an IPFS Subdomain Gateway URL with CID+path+queryString where query string has providers', async () => {
const providers = [
'/dns4/provider-server.io/tcp/443/https',
'/dns4/provider-server.io/tcp/8000'
]
await assertMatchUrl(
`http://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm.ipfs.example.com/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
protocol: 'ipfs',
cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
query: {
format: 'tar',
provider: providers
}
}
)
})
})

describe('ipns://<peerId> URLs', () => {
Expand Down
Loading
Loading