Merge branch 'master' of https://github.com/onflow/flow-go into yurii/sync-master-to-dynamic-protocol-state

Author: durkmurder

.github/workflows/ci.yml

@@ -99,7 +99,7 @@ jobs:
           cache: true
       - name: Set Test Matrix
         id: set-test-matrix
-        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils
+        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine/access engine/collection engine/common engine/consensus engine/execution/ingestion:buildjet-8vcpu-ubuntu-2204 engine/execution/computation engine/execution engine/verification engine:buildjet-4vcpu-ubuntu-2204 fvm ledger module/dkg module:buildjet-4vcpu-ubuntu-2204 network/alsp network/test/cohort1:buildjet-16vcpu-ubuntu-2204 network/test/cohort2:buildjet-4vcpu-ubuntu-2204 network/p2p/connection network/p2p/p2pnode:buildjet-4vcpu-ubuntu-2204 network/p2p/scoring network/p2p network state storage utils
 
   unit-test:
     name: Unit Tests (${{ matrix.targets.name }})
@@ -108,8 +108,8 @@ jobs:
       fail-fast: false
       matrix:
         targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
-    # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
-    runs-on: ubuntu-20.04
+    ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
+    runs-on: ${{ matrix.targets.runner }}
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
@@ -123,8 +123,8 @@ jobs:
     - name: Run tests (${{ matrix.targets.name }})
       uses: nick-fields/retry@v2
       with:
-        timeout_minutes: 25
-        max_attempts: 3
+        timeout_minutes: 35
+        max_attempts: 5
         command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test
 
       # TODO(rbtz): re-enable when we fix exisiting races.
@@ -148,17 +148,20 @@ jobs:
             make2: unittest
             retries: 1
             race: 1
+            runner: ubuntu-latest
           - name: insecure
             make1: install-tools
             make2: test
-            retries: 3
+            retries: 5
             race: 0
+            runner: buildjet-4vcpu-ubuntu-2204
           - name: integration
             make1: install-tools
             make2: test
-            retries: 3
+            retries: 5
             race: 0
-    runs-on: ubuntu-latest
+            runner: buildjet-4vcpu-ubuntu-2204
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -174,7 +177,7 @@ jobs:
           RACE_DETECTOR: ${{ matrix.race }}
         uses: nick-fields/retry@v2
         with:
-          timeout_minutes: 25
+          timeout_minutes: 35
           max_attempts: ${{ matrix.retries }}
           # run `make2` target inside each module's root
           command: VERBOSE=1 make -C ${{ matrix.name }} ${{ matrix.make2 }}
@@ -185,27 +188,101 @@ jobs:
           flags: unittests
           name: codecov-umbrella
 
+  docker-build:
+    name: Docker Build
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          # all tags are needed for integration tests
+          fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      - name: Build relic
+        run: make crypto_setup_gopath
+      - name: Docker build
+        run: make docker-build-flow docker-build-flow-corrupt
+      - name: Save Docker images
+        run: |
+          docker save \
+          gcr.io/flow-container-registry/access:latest \
+          gcr.io/flow-container-registry/collection:latest \
+          gcr.io/flow-container-registry/consensus:latest \
+          gcr.io/flow-container-registry/execution:latest \
+          gcr.io/flow-container-registry/ghost:latest \
+          gcr.io/flow-container-registry/observer:latest \
+          gcr.io/flow-container-registry/verification:latest \
+          gcr.io/flow-container-registry/access-corrupted:latest \
+          gcr.io/flow-container-registry/execution-corrupted:latest \
+          gcr.io/flow-container-registry/verification-corrupted:latest > flow-docker-images.tar
+      - name: Cache Docker images
+        uses: actions/cache@v3
+        with:
+          path: flow-docker-images.tar
+          # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run
+          key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }}
+
   integration-test:
     name: Integration Tests
+    needs: docker-build
     strategy:
       fail-fast: false
       matrix:
-        make:
-          - make -C integration access-tests
-          - make -C integration bft-framework-tests
-          - make -C integration bft-protocol-tests
-          - make -C integration bft-gossipsub-tests
-          - make -C integration collection-tests
-          - make -C integration consensus-tests
-          - make -C integration epochs-cohort1-tests
-          - make -C integration epochs-cohort2-tests
-          - make -C integration execution-tests
-          - make -C integration ghost-tests
-          - make -C integration mvp-tests
-          - make -C integration network-tests
-          - make -C integration verification-tests
-          - make -C integration upgrades-tests
-    runs-on: ubuntu-latest
+        include:
+          - name: Access Cohort1 Integration Tests
+            make: make -C integration access-cohort1-tests
+            runner: buildjet-4vcpu-ubuntu-2204
+          - name: Access Cohort2 Integration Tests
+            make: make -C integration access-cohort2-tests
+            runner: ubuntu-latest
+          - name: Access Cohort3 Integration Tests
+            make: make -C integration access-cohort3-tests
+            runner: ubuntu-latest
+            # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed
+#          - name: BFT (Framework) Integration Tests
+#            make: make -C integration bft-framework-tests
+#            runner: ubuntu-latest
+          - name: BFT (Protocol) Integration Tests
+            make: make -C integration bft-protocol-tests
+            runner: buildjet-8vcpu-ubuntu-2204
+          - name: BFT (Gossipsub) Integration Tests
+            make: make -C integration bft-gossipsub-tests
+            runner: ubuntu-latest
+          - name: Collection Integration Tests
+            make: make -C integration collection-tests
+            runner: ubuntu-latest
+          - name: Consensus Integration Tests
+            make: make -C integration consensus-tests
+            runner: ubuntu-latest
+          - name: Epoch Cohort1 Integration Tests
+            make: make -C integration epochs-cohort1-tests
+            runner: buildjet-8vcpu-ubuntu-2204
+          - name: Epoch Cohort2 Integration Tests
+            make: make -C integration epochs-cohort2-tests
+            runner: buildjet-4vcpu-ubuntu-2204
+          - name: Execution Integration Tests
+            make: make -C integration execution-tests
+            runner: ubuntu-latest
+          - name: Ghost Integration Tests
+            make: make -C integration ghost-tests
+            runner: ubuntu-latest
+          - name: MVP Integration Tests
+            make: make -C integration mvp-tests
+            runner: ubuntu-latest
+          - name: Network Integration Tests
+            make: make -C integration network-tests
+            runner: ubuntu-latest
+          - name: Verification Integration Tests
+            make: make -C integration verification-tests
+            runner: ubuntu-latest
+          - name: Upgrade Integration Tests
+            make: make -C integration upgrades-tests
+            runner: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
@@ -219,51 +296,20 @@ jobs:
         cache: true
     - name: Build relic
       run: make crypto_setup_gopath
-    - name: Docker build
-      run: make docker-build-flow docker-build-flow-corrupt
-    - name: Run tests
+    - name: Load cached Docker images
+      uses: actions/cache@v3
+      with:
+        path: flow-docker-images.tar
+        # use the same cache key as the docker-build job
+        key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }}
+    - name: Load Docker images
+      run: docker load -i flow-docker-images.tar
+    - name: Run tests (${{ matrix.name }})
       # TODO(rbtz): re-enable when we fix exisiting races.
       #env:
       #  RACE_DETECTOR: 1
       uses: nick-fields/retry@v2
       with:
-        timeout_minutes: 25
-        max_attempts: 3
+        timeout_minutes: 35
+        max_attempts: 5
         command: VERBOSE=1 ${{ matrix.make }}
-
-  localnet-test:
-    name: Localnet Compatibility Tests With Flow-CLI Client and Observer
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout repo
-      uses: actions/checkout@v3
-    - name: Setup Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ env.GO_VERSION }}
-        cache: true
-    - name: Build relic and other tools
-      run: make install-tools
-    - name: Install Flow Client In Docker
-      # This proved to be more reliable than installing it locally.
-      run: cd integration/localnet && sh client/client.sh
-    - name: Set up Localnet
-      run: bash -c 'cd integration/localnet/ && make -e OBSERVER=2 bootstrap && make start-flow'
-    - name: Ensure Observer is started
-      run: docker ps -f name=localnet-observer_1-1 | grep localnet-observer
-    - name: Get Client Version ensuring the client is provisioned
-      run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer version
-    - name: Wait for a default waiting period until a clean state
-      # This will not cause flakiness.
-      # The waiting time is a reasonable time to expect an observer to be responsive
-      run: sleep 10
-    - name: Get Status ensuring the access endpoint is online
-      run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n access status
-    - name: Wait for finalized blocks and check them
-      run: docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer blocks get latest
-    - name: Wait for finalized blocks and check them with Observer
-      run: sleep 5 && docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n access blocks get latest && docker run --network host localnet-client /go/flow -f /go/flow-localnet.json -n observer blocks get latest
-    - name: Stop localnet
-      run: bash -c 'cd integration/localnet/ && make stop'

.github/workflows/flaky-test-monitor.yml

@@ -8,7 +8,6 @@ on:
   push:
     paths:
       - '.github/workflows/flaky-test-monitor.yml'
-      - '.github/workflows/ci.yml'
 
 env:
   BIGQUERY_DATASET: production_src_flow_test_metrics

.github/workflows/tools.yml

@@ -33,8 +33,10 @@ jobs:
       with:
         go-version: "1.20"
     - name: Checkout repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
+        # to accurately get the version tag
+        fetch-depth: 0
         ref: ${{ inputs.tag }}
     - name: Build relic
       run: make crypto_setup_gopath

Makefile

@@ -52,7 +52,17 @@ cmd/util/util:
 
 .PHONY: update-core-contracts-version
 update-core-contracts-version:
+	# updates the core-contracts version in all of the go.mod files
+	# usage example: CC_VERSION=0.16.0 make update-core-contracts-version
 	./scripts/update-core-contracts.sh $(CC_VERSION)
+	make tidy
+
+.PHONY: update-cadence-version
+update-cadence-version:
+	# updates the cadence version in all of the go.mod files
+	# usage example: CC_VERSION=0.16.0 make update-cadence-version
+	./scripts/update-cadence.sh $(CC_VERSION)
+	make tidy
 
 ############################################################################################
 # CAUTION: DO NOT MODIFY THESE TARGETS! DOING SO WILL BREAK THE FLAKY TEST MONITOR
@@ -162,6 +172,7 @@ generate-mocks: install-mock-generators
 	mockery --name 'ComputationManager' --dir=engine/execution/computation --case=underscore --output="engine/execution/computation/mock" --outpkg="mock"
 	mockery --name '.*' --dir=engine/execution/computation/query --case=underscore --output="engine/execution/computation/query/mock" --outpkg="mock"
 	mockery --name 'EpochComponentsFactory' --dir=engine/collection/epochmgr --case=underscore --output="engine/collection/epochmgr/mock" --outpkg="mock"
+	mockery --name '.*' --dir=engine/execution/ --case=underscore --output="engine/execution/mock" --outpkg="mock"
 	mockery --name 'Backend' --dir=engine/collection/rpc --case=underscore --output="engine/collection/rpc/mock" --outpkg="mock"
 	mockery --name 'ProviderEngine' --dir=engine/execution/provider --case=underscore --output="engine/execution/provider/mock" --outpkg="mock"
 	(cd ./crypto && mockery --name 'PublicKey' --case=underscore --output="../module/mock" --outpkg="mock")
@@ -202,6 +213,7 @@ generate-mocks: install-mock-generators
 	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
 	mockery --name '.*' --dir=./cmd/util/ledger/reporters --case=underscore --output="./cmd/util/ledger/reporters/mock" --outpkg="mock"
 	mockery --name 'Storage' --dir=module/executiondatasync/tracker --case=underscore --output="module/executiondatasync/tracker/mock" --outpkg="mocktracker"
+	mockery --name 'ScriptExecutor' --dir=module/execution --case=underscore --output="module/execution/mock" --outpkg="mock"
 
 	#temporarily make insecure/ a non-module to allow mockery to create mocks
 	mv insecure/go.mod insecure/go2.mod
@@ -420,7 +432,7 @@ docker-build-ghost-debug:
 
 PHONY: docker-build-bootstrap
 docker-build-bootstrap:
-	docker build -f cmd/Dockerfile  --build-arg TARGET=./cmd/bootstrap --build-arg GOARCH=$(GOARCH) --target production \
+	docker build -f cmd/Dockerfile  --build-arg TARGET=./cmd/bootstrap --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --target production \
 		--label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \
 		-t "$(CONTAINER_REGISTRY)/bootstrap:latest" -t "$(CONTAINER_REGISTRY)/bootstrap:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/bootstrap:$(IMAGE_TAG)" .
 
@@ -620,7 +632,7 @@ docker-all-tools: tool-util tool-remove-execution-fork
 
 PHONY: docker-build-util
 docker-build-util:
-	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --target production \
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --target production \
 		-t "$(CONTAINER_REGISTRY)/util:latest" -t "$(CONTAINER_REGISTRY)/util:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .
 
 PHONY: tool-util
@@ -629,7 +641,7 @@ tool-util: docker-build-util
 
 PHONY: docker-build-remove-execution-fork
 docker-build-remove-execution-fork:
-	docker build -f cmd/Dockerfile --ssh default --build-arg TARGET=./cmd/util/cmd/remove-execution-fork --build-arg GOARCH=$(GOARCH) --target production \
+	docker build -f cmd/Dockerfile --ssh default --build-arg TARGET=./cmd/util/cmd/remove-execution-fork --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --target production \
 		-t "$(CONTAINER_REGISTRY)/remove-execution-fork:latest" -t "$(CONTAINER_REGISTRY)/remove-execution-fork:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/remove-execution-fork:$(IMAGE_TAG)" .
 
 PHONY: tool-remove-execution-fork

access/api.go

@@ -29,9 +29,9 @@ type API interface {
 	SendTransaction(ctx context.Context, tx *flow.TransactionBody) error
 	GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error)
 	GetTransactionsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*flow.TransactionBody, error)
-	GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*TransactionResult, error)
-	GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32) (*TransactionResult, error)
-	GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier) ([]*TransactionResult, error)
+	GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) (*TransactionResult, error)
+	GetTransactionResultByIndex(ctx context.Context, blockID flow.Identifier, index uint32, requiredEventEncodingVersion entities.EventEncodingVersion) (*TransactionResult, error)
+	GetTransactionResultsByBlockID(ctx context.Context, blockID flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) ([]*TransactionResult, error)
 
 	GetAccount(ctx context.Context, address flow.Address) (*flow.Account, error)
 	GetAccountAtLatestBlock(ctx context.Context, address flow.Address) (*flow.Account, error)
@@ -41,8 +41,8 @@ type API interface {
 	ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error)
 	ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error)
 
-	GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error)
-	GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error)
+	GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64, requiredEventEncodingVersion entities.EventEncodingVersion) ([]flow.BlockEvents, error)
+	GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier, requiredEventEncodingVersion entities.EventEncodingVersion) ([]flow.BlockEvents, error)
 
 	GetLatestProtocolStateSnapshot(ctx context.Context) ([]byte, error)