diff --git a/cmd/gateway/serve.go b/cmd/gateway/serve.go index 4657a417..37539784 100644 --- a/cmd/gateway/serve.go +++ b/cmd/gateway/serve.go @@ -40,6 +40,9 @@ import ( const ( // port is the default port to run the gateway on. port = 3000 + // routingPort is the default port for the delegated routing server (HTTP, no TLS). + // Set to 0 to disable. + routingPort = 0 // blockCacheCapacity defines the default number of blocks to cache in memory. // Blocks are typically <1MB due to IPFS chunking, so an upper bound for how // much memory the cache will utilize is approximately this number multiplied @@ -201,6 +204,41 @@ var serveCmd = &cobra.Command{ e.GET("/ipfs/*", echo.WrapHandler(ipfsHandler)) } + // Routing server (HTTP, no TLS) - for local Kubo delegated routing + var r *echo.Echo + if cfg.Gateway.RoutingPort != 0 { + // Routing handlers - returns the gateway address for content retrieval + r = echo.New() + r.HideBanner = true + r.HidePort = true + r.Use(requestLogger(log)) + r.Use(middleware.Recover()) + r.GET("/routing/v1/providers/*", func(c echo.Context) error { + return c.JSONBlob(http.StatusOK, []byte(fmt.Sprintf(`{ + "Providers": [ + { + "Schema": "peer", + "Protocols": ["transport-ipfs-gateway-http"], + "ID": "k51qzi5uqu5dj26lryc36mobgexftham120h0nu7o4ig6xu56y4h8wdvc4le6t", + "Addrs": ["/dns4/localhost/tcp/%d/tls/http"] + } + ] + }`, cfg.Gateway.Port))) + }) + r.GET("/routing/v1/peers/*", func(c echo.Context) error { + return c.JSONBlob(http.StatusOK, []byte(fmt.Sprintf(`{ + "Peers": [ + { + "Schema": "peer", + "Protocols": ["transport-ipfs-gateway-http"], + "ID": "k51qzi5uqu5dj26lryc36mobgexftham120h0nu7o4ig6xu56y4h8wdvc4le6t", + "Addrs": ["/dns4/localhost/tcp/%d/tls/http"] + } + ] + }`, cfg.Gateway.Port))) + }) + } + // print banner after short delay to ensure it only appears if no errors // occurred during startup timer := time.NewTimer(time.Second) @@ -214,22 +252,57 @@ var serveCmd = &cobra.Command{ cmd.Println(banner(build.Version, cfg.Gateway.Port, c.DID(), spaces, hosts)) }() - // shut down the server gracefully on context cancellation - go func() { - <-cmd.Context().Done() - cmd.Println("\nShutting down server...") + errCh := make(chan error, 2) + + // shutdown shuts down all servers gracefully + shutdown := func() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() if err := e.Shutdown(ctx); err != nil { cmd.PrintErrf("shutting down server: %s", err.Error()) } + if r != nil { + if err := r.Shutdown(ctx); err != nil { + cmd.PrintErrf("shutting down routing server: %s", err.Error()) + } + } + } + + // Start routing server in background if configured + if r != nil { + go func() { + routingAddr := fmt.Sprintf(":%d", cfg.Gateway.RoutingPort) + log.Infow("starting routing server", "addr", routingAddr) + if err := r.Start(routingAddr); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- fmt.Errorf("routing server: %w", err) + } + }() + } + + // Start main gateway server in background + go func() { + addr := fmt.Sprintf(":%d", cfg.Gateway.Port) + if cfg.Gateway.TlsCert != "" && cfg.Gateway.TlsKey != "" { + if err := e.StartTLS(addr, cfg.Gateway.TlsCert, cfg.Gateway.TlsKey); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- fmt.Errorf("gateway server: %w", err) + } + } else { + if err := e.Start(addr); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- fmt.Errorf("gateway server: %w", err) + } + } }() - addr := fmt.Sprintf(":%d", cfg.Gateway.Port) - if err := e.Start(addr); err != nil && !errors.Is(err, http.ErrServerClosed) { - return fmt.Errorf("closing server: %w", err) + // Wait for context cancellation or server error + select { + case <-cmd.Context().Done(): + cmd.Println("\nShutting down server...") + shutdown() + return nil + case err := <-errCh: + shutdown() + return err } - return nil }, } @@ -240,6 +313,9 @@ func init() { serveCmd.Flags().IntP("port", "p", port, "Port to run the HTTP server on") cobra.CheckErr(viper.BindPFlag("gateway.port", serveCmd.Flags().Lookup("port"))) + serveCmd.Flags().Int("routing-port", routingPort, "Port for delegated routing server (HTTP, no TLS). Set to 0 to disable.") + cobra.CheckErr(viper.BindPFlag("gateway.routing-port", serveCmd.Flags().Lookup("routing-port"))) + serveCmd.Flags().BoolP("subdomain", "s", subdomainEnabled, "Enabled subdomain gateway mode (e.g. .ipfs.)") cobra.CheckErr(viper.BindPFlag("gateway.subdomain.enabled", serveCmd.Flags().Lookup("subdomain"))) @@ -252,6 +328,12 @@ func init() { serveCmd.Flags().String("log-level", "", "Logging level for the gateway server (debug, info, warn, error)") cobra.CheckErr(viper.BindPFlag("gateway.log_level", serveCmd.Flags().Lookup("log-level"))) + serveCmd.Flags().String("tls-cert", "", "Path to TLS certificate file (enables HTTPS)") + cobra.CheckErr(viper.BindPFlag("gateway.tls-cert", serveCmd.Flags().Lookup("tls-cert"))) + + serveCmd.Flags().String("tls-key", "", "Path to TLS key file (enables HTTPS)") + cobra.CheckErr(viper.BindPFlag("gateway.tls-key", serveCmd.Flags().Lookup("tls-key"))) + GatewayCmd.AddCommand(serveCmd) } diff --git a/pkg/config/gateway.go b/pkg/config/gateway.go index deb1aa8f..0e75b9b8 100644 --- a/pkg/config/gateway.go +++ b/pkg/config/gateway.go @@ -5,6 +5,13 @@ import "errors" type GatewayConfig struct { // Port is the port to run the gateway on. Port int `mapstructure:"port" flag:"port" toml:"port"` + // TlsCert is the path to the TLS certificate file. If empty, TLS is disabled. + TlsCert string `mapstructure:"tls-cert" flag:"tls-cert" toml:"tls-cert"` + // TlsKey is the path to the TLS key file. If empty, TLS is disabled. + TlsKey string `mapstructure:"tls-key" flag:"tls-key" toml:"tls-key"` + // RoutingPort is the port to use for delegated routing requests. Use 0 to + // disable delegated routing. + RoutingPort int `mapstructure:"routing-port" flag:"routing-port" toml:"routing-port"` // BlockCacheCapacity defines the number of blocks to cache in memory. Blocks // are typically <1MB due to IPFS chunking, so an upper bound for how much // memory the cache will utilize is approximately this number multiplied by diff --git a/test/.gitignore b/test/.gitignore index c9a32716..d43a4627 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1 +1,3 @@ -doupload-dir \ No newline at end of file +doupload-dir +ipfs +mprocs.log \ No newline at end of file diff --git a/test/doupload b/test/doupload index 312c632a..f9480e65 100755 --- a/test/doupload +++ b/test/doupload @@ -150,6 +150,14 @@ main () { diff -r "$dataDir/small-files" "$outDir2" diff -r "$dataDir" "$outDir3" echo "✅ Data verified!" + + jq -n \ + --arg account "$account" \ + --arg space "$space" \ + --arg rootCID "$rootCID" \ + --arg dataDir "$dataDir" \ + --arg subdir "subdir" \ + '{$account, $space, $rootCID, $dataDir, $subdir}' > "$sandbox/test-params.json" } log_in() { diff --git a/test/gatewayretrieve b/test/gatewayretrieve new file mode 100755 index 00000000..9eb35af0 --- /dev/null +++ b/test/gatewayretrieve @@ -0,0 +1,98 @@ +#!/bin/zsh + +set -e +set -o pipefail + +export STORACHA_SERVICE_URL="https://staging.up.warm.storacha.network" +export STORACHA_SERVICE_DID="did:web:staging.up.warm.storacha.network" +export STORACHA_RECEIPTS_URL="https://staging.up.warm.storacha.network/receipt/" +export STORACHA_INDEXING_SERVICE_URL="https://staging.indexer.warm.storacha.network" +export STORACHA_INDEXING_SERVICE_DID="did:web:staging.indexer.warm.storacha.network" + +# Check for dependencies +if ! command -v jq &> /dev/null; then + echo "jq could not be found, please install it to run this script." + exit 1 +fi +if ! command -v ipfs &> /dev/null; then + echo "ipfs (Kubo) could not be found, please install it to run this script." + exit 1 +fi + +# Change to the directory of this script +cd "$(dirname "$0")" + +export IPFS_PATH=$PWD/ipfs + +sandbox="doupload-dir" + +guppy=("go" "run" ".." "--guppy-dir" "./$sandbox/storacha") + +certFile="$sandbox/cert.pem" +keyFile="$sandbox/key.pem" +outDir="$sandbox/out-kubo" + +if ! [[ -f "$sandbox/test-params.json" ]]; then + echo "Test parameters not found! Please run test/doupload first." + exit 1 +fi + +{ + read -d '' account + read -d '' space + read -d '' rootCID + read -d '' dataDir +} < <(jq --raw-output0 '.account, .space, .rootCID, .dataDir' "$sandbox/test-params.json") + +mprocsPort=4050 +gupwayPort=3000 +gupwayRoutingPort=3001 + +rm -rf "$IPFS_PATH" +ipfs init +ipfs config --json Routing "$(jq -n \ + --arg gupwayRoutingPort "$gupwayRoutingPort" \ + '{"Type": "delegated", "DelegatedRouters": ["http://localhost:\($gupwayRoutingPort)"]}' +)" +ipfs config --bool Provide.Enabled false +ipfs config --bool HTTPRetrieval.TLSInsecureSkipVerify true + +openssl req -x509 -newkey rsa:2048 -keyout "$keyFile" -out "$certFile" \ + -days 36500 -nodes -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" \ + -quiet + +echo "📥 Retrieving data through gateway and ipfs" + +config=$(mktemp) && mv "$config" "$config.json" && config="$config.json" +jq -n \ + --arg mprocsServer "127.0.0.1:$mprocsPort" \ + --arg guppy "${guppy[*]}" \ + --arg gupwayPort "$gupwayPort" \ + --arg certFile "$certFile" \ + --arg keyFile "$keyFile" \ + --arg gupwayRoutingPort "$gupwayRoutingPort" \ + --arg space "$space" \ + --arg rootCID "$rootCID" \ + --arg outDir "$outDir" \ + '{ + server: "\($mprocsServer)", + procs: { + gupway: { + shell: "\($guppy) gateway serve --port=\($gupwayPort) --trusted=false --tls-cert=\($certFile) --tls-key=\($keyFile) --routing-port=\($gupwayRoutingPort) \($space)" + }, + "kubo-daemon": { + shell: "sleep 2 && echo $SSL_CERT_FILE && mprocs --server \"\($mprocsServer)\" --ctl \"{c: select-proc, index: 1}\" && ipfs daemon" + }, + "kubo-get": { + shell: "sleep 4 && mprocs --server \"\($mprocsServer)\" --ctl \"{c: select-proc, index: 2}\" && echo \"Getting /ipfs/\($rootCID)\" && ipfs get /ipfs/\($rootCID) -o \($outDir) && mprocs --server \"\($mprocsServer)\" --ctl \"{c: quit}\"" + } + }, + }' > "$config" + +mprocs --config "$config" +rm "$config" + +echo "↔️ Verifying retrieved data matches original" +diff -r "$dataDir" "$outDir" +echo "✅ Data verified!"