Skip to content

Commit 7866fbd

Browse files
LexLuthrLexLuthr
committed
add indexer remove all
1 parent 85004d5 commit 7866fbd

File tree

18 files changed

+209
-8
lines changed

18 files changed

+209
-8
lines changed

Diff for: ‎api/api.go

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type Boost interface {
2626
Net
2727

2828
// MethodGroup: Boost
29+
BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin
2930
BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
3031
BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin
3132
BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin

Diff for: ‎api/proxy_gen.go

+13
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎build/openrpc/boost.json.gz

63 Bytes
Binary file not shown.

Diff for: ‎cmd/boostd/index.go

+27
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
2222
indexProvAnnounceLatestHttp,
2323
indexProvAnnounceDealRemovalAd,
2424
indexProvAnnounceDeal,
25+
indexProvRemoveAllCmd,
2526
},
2627
}
2728

@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
303304
return nil
304305
},
305306
}
307+
308+
var indexProvRemoveAllCmd = &cli.Command{
309+
Name: "remove-all",
310+
Usage: "Announce all removal ad for all contextIDs",
311+
Flags: []cli.Flag{},
312+
Action: func(cctx *cli.Context) error {
313+
ctx := lcli.ReqContext(cctx)
314+
315+
// get boost api
316+
napi, closer, err := bcli.GetBoostAPI(cctx)
317+
if err != nil {
318+
return err
319+
}
320+
defer closer()
321+
322+
// announce markets and boost deals
323+
cids, err := napi.BoostIndexerRemoveAll(ctx)
324+
if err != nil {
325+
return err
326+
}
327+
for _, c := range cids {
328+
fmt.Println("Published the removal ad with CID", c.String())
329+
}
330+
return nil
331+
},
332+
}

Diff for: ‎documentation/en/api-v1-methods.md

+16-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
* [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp)
2121
* [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal)
2222
* [BoostIndexerListMultihashes](#boostindexerlistmultihashes)
23+
* [BoostIndexerRemoveAll](#boostindexerremoveall)
2324
* [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid)
2425
* [BoostOfflineDealWithData](#boostofflinedealwithdata)
2526
* [I](#i)
@@ -374,7 +375,7 @@ Response:
374375
```
375376

376377
### BoostIndexerAnnounceAllDeals
377-
There are not yet any comments for this method.
378+
378379

379380
Perms: admin
380381

@@ -513,6 +514,20 @@ Response:
513514
]
514515
```
515516

517+
### BoostIndexerRemoveAll
518+
There are not yet any comments for this method.
519+
520+
Perms: admin
521+
522+
Inputs: `null`
523+
524+
Response:
525+
```json
526+
[
527+
null
528+
]
529+
```
530+
516531
### BoostLegacyDealByProposalCid
517532

518533

Diff for: ‎gql/resolver_ipni.go

+4
Original file line numberDiff line numberDiff line change
@@ -229,3 +229,7 @@ func (r *resolver) IpniDistanceFromLatestAd(ctx context.Context, args struct {
229229

230230
return count, nil
231231
}
232+
233+
func (r *resolver) IpniRemovedAllAdsStatus(ctx context.Context) (bool, error) {
234+
return r.idxProvWrapper.RemoveAllStatus(ctx), nil
235+
}

Diff for: ‎gql/schema.graphql

+3
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,9 @@ type RootQuery {
652652

653653
"""Get the latest IPNI advertisemen"""
654654
ipniDistanceFromLatestAd(LatestAdcid: String!, Adcid: String!): Int!
655+
656+
"""Get the IPNI Remove All Status"""
657+
ipniRemovedAllAdsStatus: Boolean!
655658
}
656659

657660
type RootMutation {

Diff for: ‎indexprovider/wrapper.go

+86-4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"fmt"
88
"net/url"
99
"os"
10+
"time"
1011

1112
"github.com/filecoin-project/boost/lib/legacy"
1213
"github.com/filecoin-project/boost/storagemarket/types/legacytypes"
@@ -18,11 +19,11 @@ import (
1819
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
1920
chainTypes "github.com/filecoin-project/lotus/chain/types"
2021
"github.com/google/uuid"
21-
cbor "github.com/ipfs/go-ipld-cbor"
22-
"go.uber.org/fx"
23-
2422
"github.com/ipfs/go-datastore"
23+
cbor "github.com/ipfs/go-ipld-cbor"
2524
"github.com/ipld/go-ipld-prime"
25+
"github.com/ipni/go-libipni/ingest/schema"
26+
"go.uber.org/fx"
2627

2728
"github.com/filecoin-project/boost/db"
2829
bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types"
@@ -71,6 +72,7 @@ type Wrapper struct {
7172
bitswapEnabled bool
7273
httpEnabled bool
7374
stop context.CancelFunc
75+
removeAllAds bool
7476
}
7577

7678
func NewWrapper(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, directDealsDB *db.DirectDealsDB, dealsDB *db.DealsDB,
@@ -127,7 +129,11 @@ func (w *Wrapper) Start(_ context.Context) {
127129

128130
log.Info("starting index provider")
129131

130-
go w.checkForUpdates(runCtx)
132+
if w.cfg.CurioMigration.Enable {
133+
go w.tryAnnounceRemoveAll(runCtx)
134+
} else {
135+
go w.checkForUpdates(runCtx)
136+
}
131137
}
132138

133139
func (w *Wrapper) checkForUpdates(ctx context.Context) {
@@ -867,3 +873,79 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u
867873
}
868874
return annCid, err
869875
}
876+
877+
func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) {
878+
var allAds []*schema.Advertisement
879+
_, ad, err := w.prov.GetLatestAdv(ctx)
880+
if err != nil {
881+
return nil, err
882+
}
883+
allAds = append(allAds, ad)
884+
885+
prev, err := cid.Parse(ad.PreviousID.String())
886+
if err != nil {
887+
return nil, err
888+
}
889+
890+
for prev != cid.Undef {
891+
ad, err := w.prov.GetAdv(ctx, prev)
892+
if err != nil {
893+
return nil, err
894+
}
895+
896+
prev, err = cid.Parse(ad.PreviousID.String())
897+
if err != nil {
898+
return nil, err
899+
}
900+
}
901+
902+
var entryAds []*schema.Advertisement
903+
904+
for _, ad := range allAds {
905+
if !ad.IsRm {
906+
entryAds = append(entryAds, ad)
907+
}
908+
}
909+
910+
var newAds []cid.Cid
911+
912+
for _, ad := range entryAds {
913+
a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID)
914+
if err != nil {
915+
if !errors.Is(err, provider.ErrContextIDNotFound) {
916+
return nil, fmt.Errorf("failed to publish the removal ad: %w", err)
917+
}
918+
}
919+
newAds = append(newAds, a)
920+
}
921+
922+
return newAds, nil
923+
924+
}
925+
926+
func (w *Wrapper) tryAnnounceRemoveAll(ctx context.Context) {
927+
ticker := time.NewTicker(time.Minute)
928+
929+
for {
930+
select {
931+
case <-ticker.C:
932+
out, err := w.AnnounceRemoveAll(ctx)
933+
if err != nil {
934+
log.Errorw("error while announcing remove all", "err", err)
935+
continue
936+
}
937+
if len(out) > 0 {
938+
continue
939+
}
940+
log.Debugw("Cleaned up all the IPNI ads")
941+
w.removeAllAds = true
942+
return
943+
case <-ctx.Done():
944+
return
945+
}
946+
}
947+
}
948+
949+
func (w *Wrapper) RemoveAllStatus(ctx context.Context) bool {
950+
return w.removeAllAds
951+
}

Diff for: ‎node/config/def.go

+3
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,9 @@ func DefaultBoost() *Boost {
170170
MaxDealsPerPublishMsg: 8,
171171
MaxPublishDealsFee: types.MustParseFIL("0.05"),
172172
},
173+
CurioMigration: CurioMigration{
174+
Enable: false,
175+
},
173176
}
174177
return cfg
175178
}

Diff for: ‎node/config/doc_gen.go

+14
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎node/config/types.go

+6
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ type Boost struct {
5151
HttpDownload HttpDownloadConfig
5252
Retrievals RetrievalConfig
5353
IndexProvider IndexProviderConfig
54+
CurioMigration CurioMigration
5455
}
5556

5657
type WalletsConfig struct {
@@ -399,3 +400,8 @@ type DealPublishConfig struct {
399400
// The maximum fee to pay when sending the PublishStorageDeals message
400401
MaxPublishDealsFee types.FIL
401402
}
403+
404+
type CurioMigration struct {
405+
// Enable limits the Boost functionality to prepare for the migration
406+
Enable bool
407+
}

Diff for: ‎node/impl/boost.go

+4
Original file line numberDiff line numberDiff line change
@@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error {
231231
func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) {
232232
return sm.StorageProvider.GetAsk(), nil
233233
}
234+
235+
func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) {
236+
return sm.IndexProvider.AnnounceRemoveAll(ctx)
237+
}

Diff for: ‎node/modules/directdeals.go

+1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ func NewDirectDealsProvider(provAddr address.Address, cfg *config.Boost) func(lc
3434
ddpCfg := storagemarket.DDPConfig{
3535
StartEpochSealingBuffer: abi.ChainEpoch(cfg.Dealmaking.StartEpochSealingBuffer),
3636
RemoteCommp: cfg.Dealmaking.RemoteCommp,
37+
CurioMigration: cfg.CurioMigration.Enable,
3738
}
3839

3940
prov := storagemarket.NewDirectDealsProvider(ddpCfg, provAddr, fullnodeApi, secb, commpc, commpt, sps, directDealsDB, dl, piecedirectory, ip)

Diff for: ‎node/modules/storageminer.go

+1
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,7 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func(
486486
DealLogDurationDays: cfg.Dealmaking.DealLogDurationDays,
487487
StorageFilter: cfg.Dealmaking.Filter,
488488
SealingPipelineCacheTimeout: time.Duration(cfg.Dealmaking.SealingPipelineCacheTimeout),
489+
CurioMigration: cfg.CurioMigration.Enable,
489490
}
490491
dl := logs.NewDealLogger(logsDB)
491492
tspt := httptransport.New(h, dl, httptransport.NChunksOpt(cfg.HttpDownload.NChunks), httptransport.AllowPrivateIPsOpt(cfg.HttpDownload.AllowPrivateIPs))

Diff for: ‎react/src/Ipni.js

+12-3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
IpniProviderInfoQuery,
77
IpniLatestAdQuery,
88
IpniDistanceFromLatestAdQuery,
9+
IpniRemovedAllAdsStatus,
910
} from "./gql";
1011
import moment from "moment";
1112
import React, {useEffect, useState} from "react";
@@ -89,6 +90,7 @@ function ProviderIpniInfoRender(props){
8990
adcid: adCid
9091
}
9192
})
93+
const publishedRemoveAll = useQuery(IpniRemovedAllAdsStatus)
9294
return <div className="ipni-prov-info">
9395
<h3>Provider Indexer Info</h3>
9496
<div className="subtitle">
@@ -111,13 +113,14 @@ function ProviderIpniInfoRender(props){
111113
&nbsp;
112114
<span className="aux">({moment(data.LastAdvertisementTime).fromNow()} ago)</span>
113115
&nbsp;
114-
{distance.data ? <span className="aux">({distance.data.ipniDistanceFromLatestAd} behind)</span>: ''}
116+
{distance.data ?
117+
<span className="aux">({distance.data.ipniDistanceFromLatestAd} behind)</span> : ''}
115118
</td>
116119
</tr>
117120
<tr>
118121
<th>Latest Advertisement on Boost</th>
119122
<td>
120-
{lad ? <Link to={'/ipni/ad/'+lad}>{lad}</Link>: ''}
123+
{lad ? <Link to={'/ipni/ad/' + lad}>{lad}</Link> : ''}
121124
</td>
122125
</tr>
123126
<tr>
@@ -128,6 +131,12 @@ function ProviderIpniInfoRender(props){
128131
<span className="aux">({moment(data.LastErrorTime).fromNow()} ago)</span>
129132
</td>
130133
</tr>
134+
<tr>
135+
<th>Published Removal Ads for Curio Migration</th>
136+
<td>
137+
{publishedRemoveAll.data ? publishedRemoveAll.data: ''}
138+
</td>
139+
</tr>
131140
</tbody>
132141
</table>
133142
</div>
@@ -137,7 +146,7 @@ function ProviderConfig({configJson}) {
137146
const cfg = JSON.parse(configJson)
138147
return <div>
139148
<h3>Index Provider Config</h3>
140-
<ExpandableJSObject v={cfg} topLevel={false} expanded={true} key={'config'} />
149+
<ExpandableJSObject v={cfg} topLevel={false} expanded={true} key={'config'}/>
141150
</div>
142151
}
143152

0 commit comments

Comments
 (0)