filecoin-project
diff --git a/‎.circleci/config.yml
+38-1 b/‎.circleci/config.yml
+38-1
diff --git a/‎.gitignore
+3 b/‎.gitignore
+3
diff --git a/‎Makefile
+30-6 b/‎Makefile
+30-6
diff --git a/‎README.md
+2-188 b/‎README.md
+2-188
@@ -1,6 +1,6 @@
 version: 2.1
 orbs:
-  go: gotest/[email protected].13
+  go: gotest/[email protected].14
 
 executors:
   golang:
@@ -83,6 +83,19 @@ jobs:
           paths:
             - linux
 
+  lid-docker-compose:
+    description: 'Run LID integration tests'
+    machine:
+      image: ubuntu-2004:202104-01
+      resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: local index directory docker compose tests
+          command: |
+            set -x
+            make test-lid
+
   test:
     description: |
       Run go tests
@@ -98,6 +111,10 @@ jobs:
         type: string
         default: "./..."
         description: Import paths of packages to be tested.
+      cwd:
+        type: string
+        default: "."
+        description: Directory in which to run the test command
       suite:
         type: string
         default: unit
@@ -110,9 +127,16 @@ jobs:
           command: make boostci
           no_output_timeout: 30m
       - download-params
+      # Note: setup_remote_docker takes about 10s and it's only needed for
+      # local index directory tests so it may be worth refactoring so that it's
+      # only included for local index directory
+      - setup_remote_docker:
+          version: 20.10.18
+          docker_layer_caching: true
       - run:
           name: go test
           command: |
+            cd << parameters.cwd >>
             go test \
               << parameters.go-test-flags >> \
               << parameters.target >>
@@ -306,7 +330,20 @@ workflows:
           suite: itest-markets_v1_offline_deal
           target: "./itests/markets_v1_offline_deal_test.go"
 
+      - test:
+          name: test-itest-disabled_markets_v1_deal
+          suite: itest-disabled_markets_v1_deal
+          target: "./itests/disabled_markets_v1_deal_test.go"
+
       - test:
           name: test-all
           suite: all
           target: "`go list ./... | grep -v boost/itests`"
+
+      - test:
+          name: test-itest-ipni
+          suite: itest-ipni
+          target: "./itests/ipni_publish_test.go"
+
+      - lid-docker-compose
+
@@ -1,12 +1,15 @@
 /boost
 /boostx
 /boostd
+/boostd-data
+/migrate-lid
 /devnet
 /booster-http
 /booster-bitswap
 /docgen-md
 /docgen-openrpc
 extern/filecoin-ffi/rust/target
+extern/boostd-data/boostd-data
 **/*.a
 **/*.pc
 /**/*/.DS_STORE
 
@@ -7,6 +7,7 @@ unexport GOFLAGS
 
 GOCC?=go
 
+ARCH?=$(shell arch)
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
 ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
@@ -79,19 +80,35 @@ calibnet-go: build-go
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
+migrate-lid: $(BUILD_DEPS)
+	rm -f migrate-lid
+	$(GOCC) build $(GOFLAGS) -o migrate-lid ./cmd/migrate-lid
+.PHONY: migrate-lid
+BINS+=migrate-lid
+
 boostx: $(BUILD_DEPS)
 	rm -f boostx
 	$(GOCC) build $(GOFLAGS) -o boostx ./cmd/boostx
 .PHONY: boostx
 BINS+=boostx
 
 boost: $(BUILD_DEPS)
-	rm -f boost boostd boostx
+	rm -f boost
 	$(GOCC) build $(GOFLAGS) -o boost ./cmd/boost
-	$(GOCC) build $(GOFLAGS) -o boostx ./cmd/boostx
-	$(GOCC) build $(GOFLAGS) -o boostd ./cmd/boostd
 .PHONY: boost
-BINS+=boost boostx boostd
+BINS+=boost
+
+boostd: $(BUILD_DEPS)
+	rm -f boostd
+	$(GOCC) build $(GOFLAGS) -o boostd ./cmd/boostd
+.PHONY: boostd
+BINS+=boostd
+
+boostd-data:
+	$(MAKE) -C ./extern/boostd-data
+	install -C ./extern/boostd-data/boostd-data ./boostd-data
+.PHONY: boostd-data
+BINS+=boostd-data
 
 booster-http: $(BUILD_DEPS)
 	rm -f booster-http
@@ -126,7 +143,7 @@ update-react: validate-node-version
 	npm run --prefix react build
 .PHONY: react
 
-build-go: boost devnet
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
 .PHONY: build-go
 
 build: react build-go
@@ -141,6 +158,10 @@ install-boost:
 	install -C ./boost /usr/local/bin/boost
 	install -C ./boostd /usr/local/bin/boostd
 	install -C ./boostx /usr/local/bin/boostx
+	install -C ./boostd-data /usr/local/bin/boostd-data
+	install -C ./booster-http /usr/local/bin/booster-http
+	install -C ./booster-bitswap /usr/local/bin/booster-bitswap
+	install -C ./migrate-lid /usr/local/bin/migrate-lid
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet
@@ -196,7 +217,7 @@ docsgen-openrpc-boost: docsgen-openrpc-bin
 
 ## DOCKER IMAGES
 docker_user?=filecoin
-lotus_version?=v1.23.0-rc1
+lotus_version?=v1.23.2
 ffi_from_source?=0
 build_lotus?=0
 ifeq ($(build_lotus),1)
@@ -252,6 +273,9 @@ docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-b
 	docker/lotus docker/lotus-miner
 .PHONY: docker/all
 
+test-lid:
+	cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests
+
 devnet/up:
 	rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d
 
 
@@ -15,194 +15,8 @@ See the docs at [https://boost.filecoin.io](https://boost.filecoin.io/getting-st
 
 Compile and install using the instructions at the `Building and installing` section in [the docs](https://boost.filecoin.io/getting-started#building-and-installing).
 
-## Running Boost for development
+## Running Boost devnet in Docker for development
 
-To run Boost on your development machine, you will need to set up a devnet:
-
-1. Remove any existing Lotus and Boost repositories
-```
-rm -rf ~/.lotusmarkets ~/.lotus ~/.lotusminer ~/.genesis_sectors
-rm -rf ~/.boost
-```
-
-2. Build Lotus in debug mode
-
-The version of lotus needs to match the version in Boost's go.mod
-```
-cd lotus
-git checkout <tag>
-make debug
-```
-
-3. Install Lotus
-
-The devnet script uses the installed `lotus` and `lotus-miner` binaries to run the miner and daemon.
-```
-make install
-install -C ./lotus-seed /usr/local/bin/lotus-seed
-```
-
-4. Build Boost in debug mode
-
-Double check if environment variables are set:
-```
-export LIBRARY_PATH=/opt/homebrew/lib
-export PATH="$(brew --prefix coreutils)/libexec/gnubin:/usr/local/bin:$PATH"
-```
-
-Build and install
-```
-cd boost
-make debug
-make install
-```
-
-5. Start the devnet
-
-The following command will use the binaries that you built and installed above, and will run `lotus`, `lotus-miner` and `lotus-seed`. The `lotus` version must match the version in Boost's go.mod.
-```
-cd boost
-./devnet
-```
-
-The first time you run it, it will download the Filecoin proof parameters. It will take at least 10 minutes depending on your connection speed. You may need to restart the command multiple times as your terminal will probably timeout before it finishes downloading everything.
-
-The devnet isn't designed to be restartable. After it has been successfully run once, you'll have to clear out the previous data before re-running `./devnet`:
-```
-rm -rf ~/.lotusmarkets && rm -rf ~/.lotus && rm -rf ~/.lotusminer && rm -rf ~/.genesis_sectors
-```
-
-6. Wait for `lotus-miner` to come up (through the command above)
-
-Unset these variables as they interfere with the `lotus-miner` command.
-```
-unset MINER_API_INFO
-unset FULLNODE_API_INFO
-```
-
-Then repeatedly run this command until it succeeds:
-```
-lotus-miner auth api-info --perm=admin
-```
-
-7. Get the authentication tokens to connect to the lotus daemon and miner:
-
-```
-export ENV_MINER_API_INFO=`lotus-miner auth api-info --perm=admin`
-export ENV_FULLNODE_API_INFO=`lotus auth api-info --perm=admin`
-
-export MINER_API_INFO=`echo $ENV_MINER_API_INFO | awk '{split($0,a,"="); print a[2]}'`
-export FULLNODE_API_INFO=`echo $ENV_FULLNODE_API_INFO | awk '{split($0,a,"="); print a[2]}'`
-
-echo MINER_API_INFO=$MINER_API_INFO
-echo FULLNODE_API_INFO=$FULLNODE_API_INFO
-```
-
-8. Create the wallets needed for Boost
-
-```
-export DEFAULT_WALLET=`lotus wallet list | tail -1 | awk '{print $1}'`
-export COLLAT_WALLET=`lotus wallet new bls`
-export PUBMSG_WALLET=`lotus wallet new bls`
-export CLIENT_WALLET=`lotus wallet new bls`
-```
-
-9. Add funds to the wallets
-
-```
-lotus send --from $DEFAULT_WALLET $COLLAT_WALLET 10
-lotus send --from $DEFAULT_WALLET $PUBMSG_WALLET 10
-lotus send --from $DEFAULT_WALLET $CLIENT_WALLET 10
-```
-
-Run this command repeatedly until each wallet you created has 10 FIL:
-```
-lotus wallet list
-```
-
-This should take about 10 seconds.
-
-10. Set the Publish Message Wallet as a control address on the miner
-
-```
-lotus-miner actor control set --really-do-it $PUBMSG_WALLET
-```
-
-11. Add funds into the Market Actor escrow for the client and Collateral wallets
-
-```
-lotus wallet market add --from $DEFAULT_WALLET --address $CLIENT_WALLET 5
-lotus wallet market add --address $COLLAT_WALLET 5
-```
-
-12. Initialize Boost / Create Boost repository
-
-```
-boostd -vv init \
-  --api-sealer=$MINER_API_INFO \
-  --api-sector-index=$MINER_API_INFO \
-  --wallet-publish-storage-deals=$PUBMSG_WALLET \
-  --wallet-deal-collateral=$COLLAT_WALLET \
-  --max-staging-deals-bytes=2000000000
-```
-
-13. Build the Web UI
-```
-make react
-```
-
-14. Edit config to set a fixed listen address
-
-Edit `~/.boost/config.toml`
-
-Set the port in the `ListenAddresses` key to `50000`
-```
-[Libp2p]
-  ListenAddresses = ["/ip4/0.0.0.0/tcp/50000", "/ip6/::/tcp/0"]
-```
-
-15. Run Boost
-```
-boostd -vv run
-```
-
-Note the peer ID of the boost instance:
-```
-2022-06-10T09:32:28.819Z        INFO    boostd  boostd/run.go:114       Boost libp2p node listening     {"maddr": "{12D3KooWQNNWNiJ1mieEk9EHjDVF2qBc1FSjJGEzwjnMJzteApaW: [/ip4/172.17.0.2/tcp/50000 /ip4/127.0.0.1/tcp/50000]}"}
-```
-In this example: `12D3KooWQNNWNiJ1mieEk9EHjDVF2qBc1FSjJGEzwjnMJzteApaW`
-
-14. Set the peer ID and multi-address of the miner on chain
-```
-lotus-miner actor set-peer-id <peer id>
-lotus-miner actor set-addrs /ip4/127.0.0.1/tcp/50000
-```
-
-16. Open the Web UI
-
-Open http://localhost:8080 to see the Boost UI
-
-### Make a deal with Boost
-
-1. Initialize the Boost client
-```
-boost init
-```
-
-This will output the address of the wallet (it's safe to run the init command repeatedly).
-
-2. Send funds to the client wallet
-```
-lotus send --from=$DEFAULT_WALLET <client wallet> 10
-```
-
-3. Follow the guide at https://boost.filecoin.io/tutorials/how-to-store-files-with-boost-on-filecoin
-
-Note that above you already ran a command to export FULLNODE_API (and point it to your local devnet lotus daemon).
-
-Note also that the provider address is `t01000` and you will need to supply an appropriate `--storage-price` when using `boost deal` since the devnet has a minimum price. Alternatively, using "Settings" in the Boost web UI to set the deal price to zero.
-
-## Running Boost devnet in Docker
 ### Prerequisites
 * Install Docker - https://docs.docker.com/get-docker/
 
@@ -221,7 +35,7 @@ make clean docker/all ffi_from_source=1 build_lotus=1
 
 If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.20.0-rc2 build_lotus=1`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus).
 
-### Start devnet docker stack
+### Start devnet Docker stack
 
 1. Run