diff --git a/Dockerfile b/Dockerfile index bb6f9d7..ef8638f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,6 @@ MAINTAINER IPFS Stewards ENV GOPATH /go ENV SRC_PATH /go/src/github.com/ipfs/bifrost-gateway ENV BIFROST_GATEWAY_PATH /data/bifrost-gateway -ENV STRN_LOGGER_URL https://twb3qukm2i654i3tnvx36char40aymqq.lambda-url.us-west-2.on.aws ENV STRN_ORCHESTRATOR_URL https://orchestrator.strn.pl/nodes/nearby?count=1000&core=true ENV KUBO_RPC_URL https://node0.delegate.ipfs.io,https://node1.delegate.ipfs.io,https://node2.delegate.ipfs.io,https://node3.delegate.ipfs.io diff --git a/blockstore_caboose.go b/blockstore_caboose.go index 458ce3a..6c6677f 100644 --- a/blockstore_caboose.go +++ b/blockstore_caboose.go @@ -4,13 +4,20 @@ import ( "crypto/tls" "net/http" "net/url" + "os" "time" "github.com/filecoin-saturn/caboose" blockstore "github.com/ipfs/go-ipfs-blockstore" ) -const DefaultSaturnLogger = "http://set-STRN_LOGGER_URL" +const ( + EnvSaturnLogger = "STRN_LOGGER_URL" + EnvSaturnLoggerSecret = "STRN_LOGGER_SECRET" + EnvSaturnOrchestrator = "STRN_ORCHESTRATOR_URL" + + DefaultSaturnLogger = "http://set-env-variables-STRN_LOGGER_URL-and-STRN_LOGGER_SECRET" +) func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) (blockstore.Blockstore, error) { var ( @@ -33,9 +40,19 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) } } - saturnServiceClient := &http.Client{ + saturnOrchestratorClient := &http.Client{ + Timeout: caboose.DefaultSaturnRequestTimeout, + Transport: &customTransport{ + RoundTripper: &http.Transport{ + DialContext: cdns.dialWithCachedDNS, + }, + }, + } + + saturnLoggerClient := &http.Client{ Timeout: caboose.DefaultSaturnRequestTimeout, - Transport: &withUserAgent{ + Transport: &customTransport{ + AuthorizationBearerToken: os.Getenv(EnvSaturnLoggerSecret), RoundTripper: &http.Transport{ DialContext: cdns.dialWithCachedDNS, }, @@ -44,7 +61,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) saturnRetrievalClient := &http.Client{ Timeout: caboose.DefaultSaturnRequestTimeout, - Transport: &withUserAgent{ + Transport: &customTransport{ RoundTripper: &http.Transport{ // Increasing concurrency defaults from http.DefaultTransport MaxIdleConns: 1000, @@ -72,10 +89,10 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) return caboose.NewCaboose(&caboose.Config{ OrchestratorEndpoint: orchURL, - OrchestratorClient: saturnServiceClient, + OrchestratorClient: saturnOrchestratorClient, LoggingEndpoint: *loggURL, - LoggingClient: saturnServiceClient, + LoggingClient: saturnLoggerClient, LoggingInterval: 5 * time.Second, DoValidation: true, diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 26a3cb2..fc71c35 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -22,6 +22,8 @@ import ( // https://github.com/ipfs/specs/blob/main/http-gateways/TRUSTLESS_GATEWAY.md const ( + EnvProxyGateway = "PROXY_GATEWAY_URL" + DefaultProxyGateway = "http://127.0.0.1:8080" DefaultKuboPRC = "http://127.0.0.1:5001" ) @@ -45,7 +47,7 @@ func newProxyBlockStore(gatewayURL []string, cdns *cachedDNS) blockstore.Blockst gatewayURL: gatewayURL, httpClient: &http.Client{ Timeout: GetBlockTimeout, - Transport: &withUserAgent{ + Transport: &customTransport{ // Roundtripper with increased defaults than http.Transport such that retrieving // multiple blocks from a single gateway concurrently is fast. RoundTripper: &http.Transport{ diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 96d6c86..2e52f1d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -51,7 +51,8 @@ Saturn Logger used for fraud detection. ### `STRN_LOGGER_SECRET` -TODO [bifrost-gateway/issues/43](https://github.com/ipfs/bifrost-gateway/issues/43): Saturn Logger JWT used for fraud detection. +JWT token provided by Saturn CDN. Staging (testnet) and production (mainnet) +should use different tokens. ## Logging diff --git a/handlers.go b/handlers.go index 0930739..3bfe0b4 100644 --- a/handlers.go +++ b/handlers.go @@ -40,7 +40,7 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int) (*http.Server, error) { +func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS) (*http.Server, error) { // Sets up an exchange based on the given Block Store exch, err := newExchange(bs) if err != nil { @@ -60,7 +60,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC blockService := blockservice.New(cacheBlockStore, exch) // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. - routing := newProxyRouting(kuboRPC) + routing := newProxyRouting(kuboRPC, cdns) // Creates the gateway with the block service and the routing. gwAPI, err := newBifrostGateway(blockService, routing) diff --git a/http_client.go b/http_client.go new file mode 100644 index 0000000..0ab8981 --- /dev/null +++ b/http_client.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + "net/http" +) + +type customTransport struct { + http.RoundTripper + AuthorizationBearerToken string +} + +func (adt *customTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if adt.AuthorizationBearerToken != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", adt.AuthorizationBearerToken)) + } + req.Header.Add("User-Agent", userAgent) + return adt.RoundTripper.RoundTrip(req) +} diff --git a/main.go b/main.go index 5639c0f..11f6054 100644 --- a/main.go +++ b/main.go @@ -27,11 +27,8 @@ func main() { } const ( - EnvSaturnLogger = "STRN_LOGGER_URL" - EnvSaturnOrchestrator = "STRN_ORCHESTRATOR_URL" - EnvProxyGateway = "PROXY_GATEWAY_URL" - EnvKuboRPC = "KUBO_RPC_URL" - EnvBlockCacheSize = "BLOCK_CACHE_SIZE" + EnvKuboRPC = "KUBO_RPC_URL" + EnvBlockCacheSize = "BLOCK_CACHE_SIZE" ) func init() { @@ -70,8 +67,14 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, var bs blockstore.Blockstore if saturnOrchestrator != "" { - log.Printf("Saturn backend (STRN_ORCHESTRATOR_URL) at %s", saturnOrchestrator) saturnLogger := getEnv(EnvSaturnLogger, DefaultSaturnLogger) + log.Printf("Saturn backend (%s) at %s", EnvSaturnOrchestrator, saturnOrchestrator) + log.Printf("Saturn logger (%s) at %s", EnvSaturnLogger, saturnLogger) + if os.Getenv(EnvSaturnLoggerSecret) == "" { + log.Printf("") + log.Printf(" WARNING: %s is not set", EnvSaturnLoggerSecret) + log.Printf("") + } bs, err = newCabooseBlockStore(saturnOrchestrator, saturnLogger, cdns) if err != nil { return err @@ -83,7 +86,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n") } - gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize) + gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns) if err != nil { return err } @@ -103,9 +106,9 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Printf("%s: %d", EnvBlockCacheSize, blockCacheSize) log.Printf("Legacy RPC at /api/v0 (%s) provided by %s", EnvKuboRPC, strings.Join(kuboRPC, " ")) log.Printf("Path gateway listening on http://127.0.0.1:%d", gatewayPort) + log.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", gatewayPort) log.Printf("Subdomain gateway configured on dweb.link and http://localhost:%d", gatewayPort) - log.Printf("Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", gatewayPort) - log.Printf("Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort) + log.Printf(" Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort) err := gatewaySrv.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { log.Printf("Failed to start gateway: %s", err) diff --git a/routing.go b/routing.go index 491f5cc..e724557 100644 --- a/routing.go +++ b/routing.go @@ -28,15 +28,23 @@ type proxyRouting struct { rand *rand.Rand } -func newProxyRouting(kuboRPC []string) routing.ValueStore { +func newProxyRouting(kuboRPC []string, cdns *cachedDNS) routing.ValueStore { s := rand.NewSource(time.Now().Unix()) rand := rand.New(s) return &proxyRouting{ kuboRPC: kuboRPC, httpClient: &http.Client{ - Transport: &withUserAgent{ - RoundTripper: http.DefaultTransport, + Transport: &customTransport{ + // Roundtripper with increased defaults than http.Transport such that retrieving + // multiple lookups concurrently is fast. + RoundTripper: &http.Transport{ + MaxIdleConns: 1000, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + DialContext: cdns.dialWithCachedDNS, + }, }, }, rand: rand, diff --git a/version.go b/version.go index 03a8e68..b9982fb 100644 --- a/version.go +++ b/version.go @@ -1,7 +1,6 @@ package main import ( - "net/http" "runtime/debug" "time" ) @@ -38,12 +37,3 @@ func buildVersion() string { } return "dev-build" } - -type withUserAgent struct { - http.RoundTripper -} - -func (adt *withUserAgent) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Add("User-Agent", userAgent) - return adt.RoundTripper.RoundTrip(req) -} diff --git a/version_test.go b/version_test.go index f21c7d4..35bb854 100644 --- a/version_test.go +++ b/version_test.go @@ -10,7 +10,7 @@ import ( func TestWithUserAgent(t *testing.T) { client := &http.Client{ - Transport: &withUserAgent{ + Transport: &customTransport{ RoundTripper: http.DefaultTransport, }, }