feat: support STRN_LOGGER_SECRET

- local dev works without it, but clear error is logged during startup - removed hardcoded logger endpoint to avoid 403s, and set fake one for now
ipfs-inactive · Feb 24, 2023 · 8c5a3c6 · 8c5a3c6
1 parent c305b3b
commit 8c5a3c6
Show file tree

Hide file tree

Showing 10 changed files with 73 additions and 34 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -44,7 +44,6 @@ MAINTAINER IPFS Stewards <[email protected]>
 ENV GOPATH                 /go
 ENV SRC_PATH               /go/src/github.com/ipfs/bifrost-gateway
 ENV BIFROST_GATEWAY_PATH   /data/bifrost-gateway
-ENV STRN_LOGGER_URL        https://twb3qukm2i654i3tnvx36char40aymqq.lambda-url.us-west-2.on.aws
 ENV STRN_ORCHESTRATOR_URL  https://orchestrator.strn.pl/nodes/nearby?count=1000&core=true
 ENV KUBO_RPC_URL           https://node0.delegate.ipfs.io,https://node1.delegate.ipfs.io,https://node2.delegate.ipfs.io,https://node3.delegate.ipfs.io
 

diff --git a/blockstore_caboose.go b/blockstore_caboose.go
@@ -4,13 +4,20 @@ import (
 	"crypto/tls"
 	"net/http"
 	"net/url"
+	"os"
 	"time"
 
 	"github.com/filecoin-saturn/caboose"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
 
-const DefaultSaturnLogger = "http://set-STRN_LOGGER_URL"
+const (
+	EnvSaturnLogger       = "STRN_LOGGER_URL"
+	EnvSaturnLoggerSecret = "STRN_LOGGER_SECRET"
+	EnvSaturnOrchestrator = "STRN_ORCHESTRATOR_URL"
+
+	DefaultSaturnLogger = "http://set-env-variables-STRN_LOGGER_URL-and-STRN_LOGGER_SECRET"
+)
 
 func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) (blockstore.Blockstore, error) {
 	var (
@@ -33,9 +40,19 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 		}
 	}
 
-	saturnServiceClient := &http.Client{
+	saturnOrchestratorClient := &http.Client{
+		Timeout: caboose.DefaultSaturnRequestTimeout,
+		Transport: &customTransport{
+			RoundTripper: &http.Transport{
+				DialContext: cdns.dialWithCachedDNS,
+			},
+		},
+	}
+
+	saturnLoggerClient := &http.Client{
 		Timeout: caboose.DefaultSaturnRequestTimeout,
-		Transport: &withUserAgent{
+		Transport: &customTransport{
+			AuthorizationBearerToken: os.Getenv(EnvSaturnLoggerSecret),
 			RoundTripper: &http.Transport{
 				DialContext: cdns.dialWithCachedDNS,
 			},
@@ -44,7 +61,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 
 	saturnRetrievalClient := &http.Client{
 		Timeout: caboose.DefaultSaturnRequestTimeout,
-		Transport: &withUserAgent{
+		Transport: &customTransport{
 			RoundTripper: &http.Transport{
 				// Increasing concurrency defaults from http.DefaultTransport
 				MaxIdleConns:        1000,
@@ -72,10 +89,10 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 
 	return caboose.NewCaboose(&caboose.Config{
 		OrchestratorEndpoint: orchURL,
-		OrchestratorClient:   saturnServiceClient,
+		OrchestratorClient:   saturnOrchestratorClient,
 
 		LoggingEndpoint: *loggURL,
-		LoggingClient:   saturnServiceClient,
+		LoggingClient:   saturnLoggerClient,
 		LoggingInterval: 5 * time.Second,
 
 		DoValidation: true,

diff --git a/blockstore_proxy.go b/blockstore_proxy.go
@@ -22,6 +22,8 @@ import (
 // https://github.com/ipfs/specs/blob/main/http-gateways/TRUSTLESS_GATEWAY.md
 
 const (
+	EnvProxyGateway = "PROXY_GATEWAY_URL"
+
 	DefaultProxyGateway = "http://127.0.0.1:8080"
 	DefaultKuboPRC      = "http://127.0.0.1:5001"
 )
@@ -45,7 +47,7 @@ func newProxyBlockStore(gatewayURL []string, cdns *cachedDNS) blockstore.Blockst
 		gatewayURL: gatewayURL,
 		httpClient: &http.Client{
 			Timeout: GetBlockTimeout,
-			Transport: &withUserAgent{
+			Transport: &customTransport{
 				// Roundtripper with increased defaults than http.Transport such that retrieving
 				// multiple blocks from a single gateway concurrently is fast.
 				RoundTripper: &http.Transport{

diff --git a/docs/environment-variables.md b/docs/environment-variables.md
@@ -51,7 +51,8 @@ Saturn Logger used for fraud detection.
 
 ### `STRN_LOGGER_SECRET`
 
-TODO [bifrost-gateway/issues/43](https://github.com/ipfs/bifrost-gateway/issues/43): Saturn Logger JWT used for fraud detection.
+JWT token provided by Saturn CDN. Staging (testnet) and production (mainnet)
+should use different tokens.
 
 ## Logging
 

diff --git a/handlers.go b/handlers.go
@@ -40,7 +40,7 @@ func withRequestLogger(next http.Handler) http.Handler {
 	})
 }
 
-func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int) (*http.Server, error) {
+func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS) (*http.Server, error) {
 	// Sets up an exchange based on the given Block Store
 	exch, err := newExchange(bs)
 	if err != nil {
@@ -60,7 +60,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
 	blockService := blockservice.New(cacheBlockStore, exch)
 
 	// Sets up the routing system, which will proxy the IPNS routing requests to the given gateway.
-	routing := newProxyRouting(kuboRPC)
+	routing := newProxyRouting(kuboRPC, cdns)
 
 	// Creates the gateway with the block service and the routing.
 	gwAPI, err := newBifrostGateway(blockService, routing)

diff --git a/http_client.go b/http_client.go
@@ -0,0 +1,19 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+)
+
+type customTransport struct {
+	http.RoundTripper
+	AuthorizationBearerToken string
+}
+
+func (adt *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if adt.AuthorizationBearerToken != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", adt.AuthorizationBearerToken))
+	}
+	req.Header.Add("User-Agent", userAgent)
+	return adt.RoundTripper.RoundTrip(req)
+}
diff --git a/main.go b/main.go
@@ -27,11 +27,8 @@ func main() {
 }
 
 const (
-	EnvSaturnLogger       = "STRN_LOGGER_URL"
-	EnvSaturnOrchestrator = "STRN_ORCHESTRATOR_URL"
-	EnvProxyGateway       = "PROXY_GATEWAY_URL"
-	EnvKuboRPC            = "KUBO_RPC_URL"
-	EnvBlockCacheSize     = "BLOCK_CACHE_SIZE"
+	EnvKuboRPC        = "KUBO_RPC_URL"
+	EnvBlockCacheSize = "BLOCK_CACHE_SIZE"
 )
 
 func init() {
@@ -70,8 +67,14 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`,
 		var bs blockstore.Blockstore
 
 		if saturnOrchestrator != "" {
-			log.Printf("Saturn backend (STRN_ORCHESTRATOR_URL) at %s", saturnOrchestrator)
 			saturnLogger := getEnv(EnvSaturnLogger, DefaultSaturnLogger)
+			log.Printf("Saturn backend (%s) at %s", EnvSaturnOrchestrator, saturnOrchestrator)
+			log.Printf("Saturn logger  (%s) at %s", EnvSaturnLogger, saturnLogger)
+			if os.Getenv(EnvSaturnLoggerSecret) == "" {
+				log.Printf("")
+				log.Printf("  WARNING: %s is not set", EnvSaturnLoggerSecret)
+				log.Printf("")
+			}
 			bs, err = newCabooseBlockStore(saturnOrchestrator, saturnLogger, cdns)
 			if err != nil {
 				return err
@@ -83,7 +86,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`,
 			log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n")
 		}
 
-		gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize)
+		gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns)
 		if err != nil {
 			return err
 		}
@@ -103,9 +106,9 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`,
 			log.Printf("%s: %d", EnvBlockCacheSize, blockCacheSize)
 			log.Printf("Legacy RPC at /api/v0 (%s) provided by %s", EnvKuboRPC, strings.Join(kuboRPC, " "))
 			log.Printf("Path gateway listening on http://127.0.0.1:%d", gatewayPort)
+			log.Printf("  Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", gatewayPort)
 			log.Printf("Subdomain gateway configured on dweb.link and http://localhost:%d", gatewayPort)
-			log.Printf("Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", gatewayPort)
-			log.Printf("Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort)
+			log.Printf("  Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort)
 			err := gatewaySrv.ListenAndServe()
 			if err != nil && !errors.Is(err, http.ErrServerClosed) {
 				log.Printf("Failed to start gateway: %s", err)

diff --git a/routing.go b/routing.go
@@ -28,15 +28,23 @@ type proxyRouting struct {
 	rand       *rand.Rand
 }
 
-func newProxyRouting(kuboRPC []string) routing.ValueStore {
+func newProxyRouting(kuboRPC []string, cdns *cachedDNS) routing.ValueStore {
 	s := rand.NewSource(time.Now().Unix())
 	rand := rand.New(s)
 
 	return &proxyRouting{
 		kuboRPC: kuboRPC,
 		httpClient: &http.Client{
-			Transport: &withUserAgent{
-				RoundTripper: http.DefaultTransport,
+			Transport: &customTransport{
+				// Roundtripper with increased defaults than http.Transport such that retrieving
+				// multiple lookups concurrently is fast.
+				RoundTripper: &http.Transport{
+					MaxIdleConns:        1000,
+					MaxConnsPerHost:     100,
+					MaxIdleConnsPerHost: 100,
+					IdleConnTimeout:     90 * time.Second,
+					DialContext:         cdns.dialWithCachedDNS,
+				},
 			},
 		},
 		rand: rand,

diff --git a/version.go b/version.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"net/http"
 	"runtime/debug"
 	"time"
 )
@@ -38,12 +37,3 @@ func buildVersion() string {
 	}
 	return "dev-build"
 }
-
-type withUserAgent struct {
-	http.RoundTripper
-}
-
-func (adt *withUserAgent) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.Header.Add("User-Agent", userAgent)
-	return adt.RoundTripper.RoundTrip(req)
-}
diff --git a/version_test.go b/version_test.go
@@ -10,7 +10,7 @@ import (
 
 func TestWithUserAgent(t *testing.T) {
 	client := &http.Client{
-		Transport: &withUserAgent{
+		Transport: &customTransport{
 			RoundTripper: http.DefaultTransport,
 		},
 	}