Add option to serve index provider ads over http (#1452)

* feat: option to serve index provider ads over http

* fix: config naming, hostname parsing

* fix: update docsgen

* fix: log announce address

* feat: add config for indexer direct announce urls

* refactor: always announce over pubsub

* fix: docsgen

* test: add test case for empty announce address hostname

* Add `boostd index announce-latest` command (#1456)

* feat: boostd index announce-latest

* feat: add announce-latest-http command

* fix: default direct announce url

* feat: update to index-provider v0.11.2

Author: dirkmc

api/api.go

@@ -35,6 +35,8 @@ type Boost interface {
 
 	// MethodGroup: Boost
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
+	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin
+	BoostIndexerAnnounceLatestHttp(ctx context.Context, urls []string) (cid.Cid, error)                                                         //perm:admin
 	BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin
 	BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error)                                                      //perm:admin
 	BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error)                                  //perm:admin

api/proxy_gen.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	bcli "github.com/filecoin-project/boost/cli"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/urfave/cli/v2"
@@ -11,6 +12,8 @@ var indexProvCmd = &cli.Command{
 	Usage: "Manage the index provider on Boost",
 	Subcommands: []*cli.Command{
 		indexProvAnnounceAllCmd,
+		indexProvAnnounceLatest,
+		indexProvAnnounceLatestHttp,
 	},
 }
 
@@ -31,3 +34,54 @@ var indexProvAnnounceAllCmd = &cli.Command{
 		return napi.BoostIndexerAnnounceAllDeals(ctx)
 	},
 }
+
+var indexProvAnnounceLatest = &cli.Command{
+	Name:  "announce-latest",
+	Usage: "Re-publish the latest existing advertisement to pubsub",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		c, err := napi.BoostIndexerAnnounceLatest(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Announced advertisement with cid %s\n", c)
+		return nil
+	},
+}
+
+var indexProvAnnounceLatestHttp = &cli.Command{
+	Name:  "announce-latest-http",
+	Usage: "Re-publish the latest existing advertisement to specific indexers over http",
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "announce-url",
+			Usage:    "The url(s) to announce to. If not specified, announces to the http urls in config",
+			Required: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		c, err := napi.BoostIndexerAnnounceLatestHttp(ctx, cctx.StringSlice("announce-url"))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Announced advertisement to indexers over http with cid %s\n", c)
+		return nil
+	},
+}

build/openrpc/boost.json.gz

@@ -493,7 +493,13 @@ func migrateMarketsConfig(cctx *cli.Context, mktsRepo lotus_repo.LockedRepo, boo
 		// Clear the DAG store root dir config, because the DAG store is no longer configurable in Boost
 		// (it is always at <repo path>/dagstore
 		rcfg.DAGStore.RootDir = ""
-		rcfg.IndexProvider = mktsCfg.IndexProvider
+		rcfg.IndexProvider = config.IndexProviderConfig{
+			Enable:               mktsCfg.IndexProvider.Enable,
+			EntriesCacheCapacity: mktsCfg.IndexProvider.EntriesCacheCapacity,
+			EntriesChunkSize:     mktsCfg.IndexProvider.EntriesChunkSize,
+			TopicName:            mktsCfg.IndexProvider.TopicName,
+			PurgeCacheOnStart:    mktsCfg.IndexProvider.PurgeCacheOnStart,
+		}
 		rcfg.IndexProvider.Enable = true // Enable index provider in Boost by default
 
 		if fromMonolith {

cmd/boostd/index.go

@@ -23,6 +23,8 @@
   * [BoostDealBySignedProposalCid](#boostdealbysignedproposalcid)
   * [BoostDummyDeal](#boostdummydeal)
   * [BoostIndexerAnnounceAllDeals](#boostindexerannouncealldeals)
+  * [BoostIndexerAnnounceLatest](#boostindexerannouncelatest)
+  * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
   * [BoostMakeDeal](#boostmakedeal)
   * [BoostOfflineDealWithData](#boostofflinedealwithdata)
 * [Deals](#deals)
@@ -563,6 +565,41 @@ Inputs: `null`
 
 Response: `{}`
 
+### BoostIndexerAnnounceLatest
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
+### BoostIndexerAnnounceLatestHttp
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  [
+    "string value"
+  ]
+]
+```
+
+Response:
+```json
+{
+  "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+}
+```
+
 ### BoostMakeDeal
 
 

cmd/boostd/init.go

@@ -76,7 +76,7 @@ require (
 	github.com/ipld/go-car/v2 v2.7.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/ipld/go-ipld-selector-text-lite v0.0.1
-	github.com/ipni/index-provider v0.11.1
+	github.com/ipni/index-provider v0.11.2
 	github.com/ipni/storetheindex v0.5.10
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jpillora/backoff v1.0.0

documentation/en/api-v1-methods.md

@@ -951,8 +951,8 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y=
 github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM=
-github.com/ipni/index-provider v0.11.1 h1:viNfSBvZA9G+Qe6/FGqfZtavnu4tTSfGUoWEECavqoI=
-github.com/ipni/index-provider v0.11.1/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
+github.com/ipni/index-provider v0.11.2 h1:nvykWK+/ncPTqHiuiJdXp/O0UF0V7iWesjHGKX//NYc=
+github.com/ipni/index-provider v0.11.2/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4=
 github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4=
 github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=

go.mod

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"net/url"
 	"os"
 	"path/filepath"
 
@@ -22,6 +23,7 @@ import (
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	provider "github.com/ipni/index-provider"
+	"github.com/ipni/index-provider/engine"
 	"github.com/ipni/index-provider/engine/xproviders"
 	"github.com/ipni/index-provider/metadata"
 	"github.com/libp2p/go-libp2p/core/crypto"
@@ -262,6 +264,35 @@ func (w *Wrapper) IndexerAnnounceAllDeals(ctx context.Context) error {
 	return merr
 }
 
+func (w *Wrapper) IndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) {
+	e, ok := w.prov.(*engine.Engine)
+	if !ok {
+		return cid.Undef, fmt.Errorf("index provider is disabled")
+	}
+	return e.PublishLatest(ctx)
+}
+
+func (w *Wrapper) IndexerAnnounceLatestHttp(ctx context.Context, announceUrls []string) (cid.Cid, error) {
+	e, ok := w.prov.(*engine.Engine)
+	if !ok {
+		return cid.Undef, fmt.Errorf("index provider is disabled")
+	}
+
+	if len(announceUrls) == 0 {
+		announceUrls = w.cfg.IndexProvider.Announce.DirectAnnounceURLs
+	}
+
+	urls := make([]*url.URL, 0, len(announceUrls))
+	for _, us := range announceUrls {
+		u, err := url.Parse(us)
+		if err != nil {
+			return cid.Undef, fmt.Errorf("parsing url %s: %w", us, err)
+		}
+		urls = append(urls, u)
+	}
+	return e.PublishLatestHTTP(ctx, urls...)
+}
+
 func (w *Wrapper) Start(ctx context.Context) {
 	// re-init dagstore shards for Boost deals if needed
 	if _, err := w.DagstoreReinitBoostDeals(ctx); err != nil {

go.sum

@@ -171,14 +171,25 @@ func DefaultBoost() *Boost {
 			MaxConcurrencyStorageCalls: 100,
 			GCInterval:                 lotus_config.Duration(1 * time.Minute),
 		},
-		IndexProvider: lotus_config.IndexProviderConfig{
+		IndexProvider: IndexProviderConfig{
 			Enable:               true,
 			EntriesCacheCapacity: 1024,
 			EntriesChunkSize:     16384,
 			// The default empty TopicName means it is inferred from network name, in the following
 			// format: "/indexer/ingest/<network-name>"
 			TopicName:         "",
 			PurgeCacheOnStart: false,
+
+			Announce: IndexProviderAnnounceConfig{
+				AnnounceOverHttp:   false,
+				DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"},
+			},
+
+			HttpPublisher: IndexProviderHttpPublisherConfig{
+				Enabled:        false,
+				PublicHostname: "",
+				Port:           3104,
+			},
 		},
 	}
 	return cfg