explore: Demonstrate and test retrieval through Kubo

cmd/gateway/serve.go

@@ -40,6 +40,9 @@ import (
 const (
 	// port is the default port to run the gateway on.
 	port = 3000
+	// routingPort is the default port for the delegated routing server (HTTP, no TLS).
+	// Set to 0 to disable.
+	routingPort = 0
 	// blockCacheCapacity defines the default number of blocks to cache in memory.
 	// Blocks are typically <1MB due to IPFS chunking, so an upper bound for how
 	// much memory the cache will utilize is approximately this number multiplied
@@ -201,6 +204,41 @@ var serveCmd = &cobra.Command{
 			e.GET("/ipfs/*", echo.WrapHandler(ipfsHandler))
 		}
 
+		// Routing server (HTTP, no TLS) - for local Kubo delegated routing
+		var r *echo.Echo
+		if cfg.Gateway.RoutingPort != 0 {
+			// Routing handlers - returns the gateway address for content retrieval
+			r = echo.New()
+			r.HideBanner = true
+			r.HidePort = true
+			r.Use(requestLogger(log))
+			r.Use(middleware.Recover())
+			r.GET("/routing/v1/providers/*", func(c echo.Context) error {
+				return c.JSONBlob(http.StatusOK, []byte(fmt.Sprintf(`{
+					"Providers": [
+						{
+							"Schema": "peer",
+							"Protocols": ["transport-ipfs-gateway-http"],
+							"ID": "k51qzi5uqu5dj26lryc36mobgexftham120h0nu7o4ig6xu56y4h8wdvc4le6t",
+							"Addrs": ["/dns4/localhost/tcp/%d/tls/http"]
+						}
+					]
+				}`, cfg.Gateway.Port)))
+			})
+			r.GET("/routing/v1/peers/*", func(c echo.Context) error {
+				return c.JSONBlob(http.StatusOK, []byte(fmt.Sprintf(`{
+					"Peers": [
+						{
+							"Schema": "peer",
+							"Protocols": ["transport-ipfs-gateway-http"],
+							"ID": "k51qzi5uqu5dj26lryc36mobgexftham120h0nu7o4ig6xu56y4h8wdvc4le6t",
+							"Addrs": ["/dns4/localhost/tcp/%d/tls/http"]
+						}
+					]
+				}`, cfg.Gateway.Port)))
+			})
+		}
+
 		// print banner after short delay to ensure it only appears if no errors
 		// occurred during startup
 		timer := time.NewTimer(time.Second)
@@ -214,22 +252,57 @@ var serveCmd = &cobra.Command{
 			cmd.Println(banner(build.Version, cfg.Gateway.Port, c.DID(), spaces, hosts))
 		}()
 
-		// shut down the server gracefully on context cancellation
-		go func() {
-			<-cmd.Context().Done()
-			cmd.Println("\nShutting down server...")
+		errCh := make(chan error, 2)
+
+		// shutdown shuts down all servers gracefully
+		shutdown := func() {
 			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 			defer cancel()
 			if err := e.Shutdown(ctx); err != nil {
 				cmd.PrintErrf("shutting down server: %s", err.Error())
 			}
+			if r != nil {
+				if err := r.Shutdown(ctx); err != nil {
+					cmd.PrintErrf("shutting down routing server: %s", err.Error())
+				}
+			}
+		}
+
+		// Start routing server in background if configured
+		if r != nil {
+			go func() {
+				routingAddr := fmt.Sprintf(":%d", cfg.Gateway.RoutingPort)
+				log.Infow("starting routing server", "addr", routingAddr)
+				if err := r.Start(routingAddr); err != nil && !errors.Is(err, http.ErrServerClosed) {
+					errCh <- fmt.Errorf("routing server: %w", err)
+				}
+			}()
+		}
+
+		// Start main gateway server in background
+		go func() {
+			addr := fmt.Sprintf(":%d", cfg.Gateway.Port)
+			if cfg.Gateway.TlsCert != "" && cfg.Gateway.TlsKey != "" {
+				if err := e.StartTLS(addr, cfg.Gateway.TlsCert, cfg.Gateway.TlsKey); err != nil && !errors.Is(err, http.ErrServerClosed) {
+					errCh <- fmt.Errorf("gateway server: %w", err)
+				}
+			} else {
+				if err := e.Start(addr); err != nil && !errors.Is(err, http.ErrServerClosed) {
+					errCh <- fmt.Errorf("gateway server: %w", err)
+				}
+			}
 		}()
 
-		addr := fmt.Sprintf(":%d", cfg.Gateway.Port)
-		if err := e.Start(addr); err != nil && !errors.Is(err, http.ErrServerClosed) {
-			return fmt.Errorf("closing server: %w", err)
+		// Wait for context cancellation or server error
+		select {
+		case <-cmd.Context().Done():
+			cmd.Println("\nShutting down server...")
+			shutdown()
+			return nil
+		case err := <-errCh:
+			shutdown()
+			return err
 		}
-		return nil
 	},
 }
 
@@ -240,6 +313,9 @@ func init() {
 	serveCmd.Flags().IntP("port", "p", port, "Port to run the HTTP server on")
 	cobra.CheckErr(viper.BindPFlag("gateway.port", serveCmd.Flags().Lookup("port")))
 
+	serveCmd.Flags().Int("routing-port", routingPort, "Port for delegated routing server (HTTP, no TLS). Set to 0 to disable.")
+	cobra.CheckErr(viper.BindPFlag("gateway.routing-port", serveCmd.Flags().Lookup("routing-port")))
+
 	serveCmd.Flags().BoolP("subdomain", "s", subdomainEnabled, "Enabled subdomain gateway mode (e.g. <cid>.ipfs.<gateway-host>)")
 	cobra.CheckErr(viper.BindPFlag("gateway.subdomain.enabled", serveCmd.Flags().Lookup("subdomain")))
 
@@ -252,6 +328,12 @@ func init() {
 	serveCmd.Flags().String("log-level", "", "Logging level for the gateway server (debug, info, warn, error)")
 	cobra.CheckErr(viper.BindPFlag("gateway.log_level", serveCmd.Flags().Lookup("log-level")))
 
+	serveCmd.Flags().String("tls-cert", "", "Path to TLS certificate file (enables HTTPS)")
+	cobra.CheckErr(viper.BindPFlag("gateway.tls-cert", serveCmd.Flags().Lookup("tls-cert")))
+
+	serveCmd.Flags().String("tls-key", "", "Path to TLS key file (enables HTTPS)")
+	cobra.CheckErr(viper.BindPFlag("gateway.tls-key", serveCmd.Flags().Lookup("tls-key")))
+
 	GatewayCmd.AddCommand(serveCmd)
 }
 

pkg/config/gateway.go

@@ -5,6 +5,13 @@ import "errors"
 type GatewayConfig struct {
 	// Port is the port to run the gateway on.
 	Port int `mapstructure:"port" flag:"port" toml:"port"`
+	// TlsCert is the path to the TLS certificate file. If empty, TLS is disabled.
+	TlsCert string `mapstructure:"tls-cert" flag:"tls-cert" toml:"tls-cert"`
+	// TlsKey is the path to the TLS key file. If empty, TLS is disabled.
+	TlsKey string `mapstructure:"tls-key" flag:"tls-key" toml:"tls-key"`
+	// RoutingPort is the port to use for delegated routing requests. Use 0 to
+	// disable delegated routing.
+	RoutingPort int `mapstructure:"routing-port" flag:"routing-port" toml:"routing-port"`
 	// BlockCacheCapacity defines the number of blocks to cache in memory. Blocks
 	// are typically <1MB due to IPFS chunking, so an upper bound for how much
 	// memory the cache will utilize is approximately this number multiplied by

test/.gitignore

@@ -1 +1,3 @@
-doupload-dir
+doupload-dir
+ipfs
+mprocs.log

test/doupload

@@ -150,6 +150,14 @@ main () {
 	diff -r "$dataDir/small-files" "$outDir2"
 	diff -r "$dataDir" "$outDir3"
 	echo "✅ Data verified!"
+
+	jq -n \
+		--arg account "$account" \
+		--arg space "$space" \
+		--arg rootCID "$rootCID" \
+		--arg dataDir "$dataDir" \
+		--arg subdir "subdir" \
+		'{$account, $space, $rootCID, $dataDir, $subdir}' > "$sandbox/test-params.json"
 }
 
 log_in() {

test/gatewayretrieve

@@ -0,0 +1,98 @@
+#!/bin/zsh
+
+set -e
+set -o pipefail
+
+export STORACHA_SERVICE_URL="https://staging.up.warm.storacha.network"
+export STORACHA_SERVICE_DID="did:web:staging.up.warm.storacha.network"
+export STORACHA_RECEIPTS_URL="https://staging.up.warm.storacha.network/receipt/"
+export STORACHA_INDEXING_SERVICE_URL="https://staging.indexer.warm.storacha.network"
+export STORACHA_INDEXING_SERVICE_DID="did:web:staging.indexer.warm.storacha.network"
+
+# Check for dependencies
+if ! command -v jq &> /dev/null; then
+		echo "jq could not be found, please install it to run this script."
+		exit 1
+fi
+if ! command -v ipfs &> /dev/null; then
+		echo "ipfs (Kubo) could not be found, please install it to run this script."
+		exit 1
+fi
+
+# Change to the directory of this script
+cd "$(dirname "$0")"
+
+export IPFS_PATH=$PWD/ipfs
+
+sandbox="doupload-dir"
+
+guppy=("go" "run" ".." "--guppy-dir" "./$sandbox/storacha")
+
+certFile="$sandbox/cert.pem"
+keyFile="$sandbox/key.pem"
+outDir="$sandbox/out-kubo"
+
+if ! [[ -f "$sandbox/test-params.json" ]]; then
+	echo "Test parameters not found! Please run test/doupload first."
+	exit 1
+fi
+
+{
+  read -d '' account
+  read -d '' space
+  read -d '' rootCID
+	read -d '' dataDir
+} < <(jq --raw-output0 '.account, .space, .rootCID, .dataDir' "$sandbox/test-params.json")
+
+mprocsPort=4050
+gupwayPort=3000
+gupwayRoutingPort=3001
+
+rm -rf "$IPFS_PATH"
+ipfs init
+ipfs config --json Routing "$(jq -n \
+	--arg gupwayRoutingPort "$gupwayRoutingPort" \
+	'{"Type": "delegated", "DelegatedRouters": ["http://localhost:\($gupwayRoutingPort)"]}'
+)"
+ipfs config --bool Provide.Enabled false
+ipfs config --bool HTTPRetrieval.TLSInsecureSkipVerify true
+
+openssl req -x509 -newkey rsa:2048 -keyout "$keyFile" -out "$certFile" \
+  -days 36500 -nodes -subj "/CN=localhost" \
+  -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" \
+	-quiet
+
+echo "📥 Retrieving data through gateway and ipfs"
+
+config=$(mktemp) && mv "$config" "$config.json" && config="$config.json"
+jq -n \
+	--arg mprocsServer "127.0.0.1:$mprocsPort" \
+	--arg guppy "${guppy[*]}" \
+	--arg gupwayPort "$gupwayPort" \
+	--arg certFile "$certFile" \
+	--arg keyFile "$keyFile" \
+	--arg gupwayRoutingPort "$gupwayRoutingPort" \
+	--arg space "$space" \
+	--arg rootCID "$rootCID" \
+	--arg outDir "$outDir" \
+	'{
+		server: "\($mprocsServer)",
+		procs: {
+			gupway: {
+				shell: "\($guppy) gateway serve --port=\($gupwayPort) --trusted=false --tls-cert=\($certFile) --tls-key=\($keyFile) --routing-port=\($gupwayRoutingPort) \($space)"
+			},
+			"kubo-daemon": {
+				shell: "sleep 2 && echo $SSL_CERT_FILE && mprocs --server \"\($mprocsServer)\" --ctl \"{c: select-proc, index: 1}\" && ipfs daemon"
+			},
+			"kubo-get": {
+				shell: "sleep 4 && mprocs --server \"\($mprocsServer)\" --ctl \"{c: select-proc, index: 2}\" && echo \"Getting /ipfs/\($rootCID)\" && ipfs get /ipfs/\($rootCID) -o \($outDir) && mprocs --server \"\($mprocsServer)\" --ctl \"{c: quit}\""
+			}
+		},
+	}' > "$config"
+
+mprocs --config "$config"
+rm "$config"
+
+echo "↔️ Verifying retrieved data matches original"
+diff -r "$dataDir" "$outDir"
+echo "✅ Data verified!"