Feat/sync nodes

.dockerignore

@@ -1,6 +1,6 @@
 node_modules
-/dist/*
-!/dist/dashboard
+/dist
 logs
+c2d_storage
 .env.local
 .env

.env.example

@@ -20,17 +20,24 @@ export FEE_AMOUNT=
 export ADDRESS_FILE=
 export NODE_ENV=
 export AUTHORIZED_DECRYPTERS=
+export AUTHORIZED_DECRYPTERS_LIST=
 export OPERATOR_SERVICE_URL=
+export POLICY_SERVER_URL
 export INTERFACES=
 export ALLOWED_VALIDATORS=
+export ALLOWED_VALIDATORS_LIST=
+export AUTHORIZED_PUBLISHERS=
+export AUTHORIZED_PUBLISHERS_LIST=
 export INDEXER_INTERVAL=
 export ALLOWED_ADMINS=
-export DASHBOARD=true
+export ALLOWED_ADMINS_LIST=
+export CONTROL_PANEL=true
 export RATE_DENY_LIST=
-export MAX_REQ_PER_SECOND=
+export MAX_REQ_PER_MINUTE=
 export MAX_CHECKSUM_LENGTH=
 export LOG_LEVEL=
 export HTTP_API_PORT=
+export VALIDATE_UNSIGNED_DDO=
 
 ## p2p
 
@@ -55,4 +62,9 @@ export P2P_ENABLE_AUTONAT=
 export P2P_ENABLE_CIRCUIT_RELAY_SERVER=
 export P2P_ENABLE_CIRCUIT_RELAY_CLIENT=
 export P2P_BOOTSTRAP_NODES=
-export P2P_FILTER_ANNOUNCED_ADDRESSES=
+export P2P_FILTER_ANNOUNCED_ADDRESSES=
+
+## compute
+export DOCKER_COMPUTE_ENVIRONMENTS=
+
+

.github/CODEOWNERS

@@ -1 +1 @@
-* @alexcos20
+* @alexcos20 @bogdanfazakas @jamiehewitt15 @mariacarmina @paulo-ocean @giurgiur99

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 'v20.16.0'
+          node-version: 'v20.19.0'
       - name: Cache node_modules
         uses: actions/cache@v3
         env:
@@ -42,7 +42,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        node: ['18.20.4', 'v20.16.0', 'v22.5.1']
+        node: ['18.20.4', 'v20.19.0', 'v22.5.1']
 
     steps:
       - uses: actions/checkout@v4
@@ -66,7 +66,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 'v20.16.0'
+          node-version: 'v20.19.0'
       - name: Cache node_modules
         uses: actions/cache@v3
         env:
@@ -100,7 +100,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 'v20.16.0'
+          node-version: 'v20.19.0'
       - name: Cache node_modules
         uses: actions/cache@v3
         env:
@@ -125,24 +125,30 @@ jobs:
               DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Run Barge
         working-directory: ${{ github.workspace }}/barge
+        env:
+          CONTRACTS_VERSION: escrow
         run: |
-          bash -x start_ocean.sh --no-aquarius --no-provider --no-dashboard --with-c2d --with-typesense 2>&1 > start_ocean.log &
+          bash -x start_ocean.sh --no-aquarius --no-provider --no-dashboard --with-typesense 2>&1 > start_ocean.log &
       - run: npm ci
       - run: npm run build
-      - name: Wait for contracts deployment and C2D cluster to be ready
+      - run: docker image ls
+      - name: Delete default runner images
+        run: |
+          rm -rf /usr/share/swift/
+      - name: Wait for contracts deployment
         working-directory: ${{ github.workspace }}/barge
         run: |
           for i in $(seq 1 250); do
-            sleep 10
+            sleep 5
             [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" ] && break
             done
       - name: docker logs
-        run: docker logs ocean-ocean-contracts-1 && docker logs ocean-kindcluster-1 && docker logs ocean-computetodata-1 && docker logs ocean-typesense-1
+        run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
         if: ${{ failure() }}
       - name: integration tests
         run: npm run test:integration:cover
         env:
-          OPERATOR_SERVICE_URL: '["http://172.15.0.13:31000"]'
+          # OPERATOR_SERVICE_URL: '["http://172.15.0.13:31000"]'
           PRIVATE_KEY: ${{ secrets.NODE1_PRIVATE_KEY }}
           NODE1_PRIVATE_KEY: ${{ secrets.NODE1_PRIVATE_KEY }}
           NODE2_PRIVATE_KEY: ${{ secrets.NODE2_PRIVATE_KEY }}
@@ -157,7 +163,7 @@ jobs:
           ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json'
           ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json'
       - name: docker logs
-        run: docker logs ocean-ocean-contracts-1 && docker logs ocean-kindcluster-1 && docker logs ocean-computetodata-1 && docker logs ocean-typesense-1
+        run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
         if: ${{ failure() }}
       - uses: actions/upload-artifact@v4
         with:
@@ -174,7 +180,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 'v20.16.0'
+          node-version: 'v20.19.0'
 
       - name: Cache node_modules
         uses: actions/cache@v3
@@ -225,7 +231,7 @@ jobs:
           done
 
       - name: docker logs
-        run: docker logs ocean-ocean-contracts-1 && docker logs ocean-kindcluster-1 && docker logs ocean-computetodata-1 && docker logs ocean-typesense-1
+        run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
         if: ${{ failure() }}
 
       - name: Checkout Ocean Node
@@ -286,3 +292,22 @@ jobs:
         run: npm run test:system
         env:
           AVOID_LOOP_RUN: true
+
+  control_panel_build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'v20.19.0'
+      - name: Cache node_modules
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-control-panel-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: ${{ runner.os }}-control-panel-${{ env.cache-name }}-
+      - run: npm ci
+      - name: Build control panel
+        run: npm run build:controlpanel

.github/workflows/docker.yml

@@ -14,6 +14,7 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    if: ${{ github.actor != 'dependabot[bot]' }}
     strategy:
       fail-fast: false
       matrix:
@@ -81,3 +82,45 @@ jobs:
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set Docker metadata
+        id: ocean_node_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            oceanprotocol/ocean-node
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=ref,event=pr
+          # type=semver,pattern={{major}}.{{minor}}
+          # type=semver,pattern={{major}}
+          # type=sha
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKER_PUSH_TOKEN }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          $(printf 'oceanprotocol/ocean-node@sha256:%s ' *)
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect oceanprotocol/ocean-node:${{ steps.ocean_node_meta.outputs.version }}

.gitignore

@@ -1,5 +1,8 @@
-# Dahsboard
-dashboard/.env
+# Control panel
+controlpanel/.env
+
+# Docker compose
+**/docker-compose.yml
 
 # Logs
 logs
@@ -94,7 +97,7 @@ out
 .nuxt
 
 /dist/*
-!/dist/dashboard
+!/dist/controlpanel
 
 # Gatsby files
 .cache/
@@ -155,3 +158,4 @@ html-report.html
 # databases
 *.sqlite
 databases/*
+c2d_storage/*

.nvmrc

@@ -1 +1 @@
-v20.16.0
+v20.19.0

CHANGELOG.md

@@ -4,8 +4,99 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.2.3](https://github.com/oceanprotocol/ocean-node/compare/v0.2.1...v0.2.3)
+
+- Update node script - C2D  [`#896`](https://github.com/oceanprotocol/ocean-node/pull/896)
+- fix docker-compose [`#895`](https://github.com/oceanprotocol/ocean-node/pull/895)
+- re-indexing old DDOs [`#867`](https://github.com/oceanprotocol/ocean-node/pull/867)
+- Upgrade tsx dep to v4.x [`#893`](https://github.com/oceanprotocol/ocean-node/pull/893)
+- C2D Docker [`#705`](https://github.com/oceanprotocol/ocean-node/pull/705)
+- Updating codeowners [`#887`](https://github.com/oceanprotocol/ocean-node/pull/887)
+- fix issue with empty  nft fields [`#886`](https://github.com/oceanprotocol/ocean-node/pull/886)
+- add allowed admins access list [`#841`](https://github.com/oceanprotocol/ocean-node/pull/841)
+- Update error message for invalid peer connection [`#874`](https://github.com/oceanprotocol/ocean-node/pull/874)
+- add AUTHORIZED_DECRYPTERS_LIST [`#836`](https://github.com/oceanprotocol/ocean-node/pull/836)
+- fix status code if policy server not available [`#869`](https://github.com/oceanprotocol/ocean-node/pull/869)
+- Fix DDO: Stats and Prices for exchanges/dispensers  [`#774`](https://github.com/oceanprotocol/ocean-node/pull/774)
+- move p2p getters as handlers [`#862`](https://github.com/oceanprotocol/ocean-node/pull/862)
+- always check remote peerId [`#864`](https://github.com/oceanprotocol/ocean-node/pull/864)
+- Test if dashboard changes are already committed [`#842`](https://github.com/oceanprotocol/ocean-node/pull/842)
+- add AUTHORIZED_PUBLISHERS_*** env variables [`#826`](https://github.com/oceanprotocol/ocean-node/pull/826)
+- Issue 814 credentials types [`#823`](https://github.com/oceanprotocol/ocean-node/pull/823)
+- remove echo command [`#839`](https://github.com/oceanprotocol/ocean-node/pull/839)
+- Issue 808 - new accesslist credentials type [`#819`](https://github.com/oceanprotocol/ocean-node/pull/819)
+- add ALLOWED_VALIDATORS_LIST [`#829`](https://github.com/oceanprotocol/ocean-node/pull/829)
+- update build files and hash [`#821`](https://github.com/oceanprotocol/ocean-node/pull/821)
+- add * as match all rule for address types [`#837`](https://github.com/oceanprotocol/ocean-node/pull/837)
+- fix: return correct message on policy server call [`#834`](https://github.com/oceanprotocol/ocean-node/pull/834)
+- add policyServerPassthrough routes [`#832`](https://github.com/oceanprotocol/ocean-node/pull/832)
+- Bump fast-xml-parser from 4.3.6 to 4.5.0 in /dashboard [`#711`](https://github.com/oceanprotocol/ocean-node/pull/711)
+- Bump secp256k1 from 5.0.0 to 5.0.1 in /dashboard [`#725`](https://github.com/oceanprotocol/ocean-node/pull/725)
+- Bump elliptic from 6.5.5 to 6.6.0 in /dashboard [`#746`](https://github.com/oceanprotocol/ocean-node/pull/746)
+- Bump cross-spawn and aegir (UPDATE: delete deps) [`#757`](https://github.com/oceanprotocol/ocean-node/pull/757)
+- fix corner case issue, cannot start indexing on ganache without ADDRESS_FILE [`#775`](https://github.com/oceanprotocol/ocean-node/pull/775)
+- add findPeer & dht refactor [`#793`](https://github.com/oceanprotocol/ocean-node/pull/793)
+- add support for identifyPush protocol [`#794`](https://github.com/oceanprotocol/ocean-node/pull/794)
+- no need to store everything in peerStore [`#795`](https://github.com/oceanprotocol/ocean-node/pull/795)
+- add multiAddrs for directCommand [`#796`](https://github.com/oceanprotocol/ocean-node/pull/796)
+- Refactor connections rate, rate per minute instead of second [`#785`](https://github.com/oceanprotocol/ocean-node/pull/785)
+- add missing check for ddo DB on republish ddos [`#789`](https://github.com/oceanprotocol/ocean-node/pull/789)
+- recalculate code hash if needed [`#790`](https://github.com/oceanprotocol/ocean-node/pull/790)
+- fix usage of getBoolEnvValue [`#784`](https://github.com/oceanprotocol/ocean-node/pull/784)
+- refactor handlers interfaces [`37cbfa5`](https://github.com/oceanprotocol/ocean-node/commit/37cbfa5076d6b665d9ba524156a8f48a656fe93f)
+- mv unit test to integration (need contracts deployment) [`7236ac5`](https://github.com/oceanprotocol/ocean-node/commit/7236ac5a603b2ef9c66ccf68a8a6d52acdb96855)
+- refactor, move getAdmins fn to auth [`5ba8bd5`](https://github.com/oceanprotocol/ocean-node/commit/5ba8bd528286a5c18acda572a1ce6170672415e6)
+
+#### [v0.2.1](https://github.com/oceanprotocol/ocean-node/compare/v0.2.0...v0.2.1)
+
+> 5 December 2024
+
+- have bootstrap config var [`#782`](https://github.com/oceanprotocol/ocean-node/pull/782)
+- Removing unused express dependency from dashboard [`#761`](https://github.com/oceanprotocol/ocean-node/pull/761)
+- check if rpcs and indexer_networks are aligned, log [`#750`](https://github.com/oceanprotocol/ocean-node/pull/750)
+- remove node 16 image references [`#759`](https://github.com/oceanprotocol/ocean-node/pull/759)
+- try debug and fix randomly logs failing test [`#752`](https://github.com/oceanprotocol/ocean-node/pull/752)
+- fix dashboard hash out of sync [`#744`](https://github.com/oceanprotocol/ocean-node/pull/744)
+- improve reindex command, optional block, also check if thread is running [`#747`](https://github.com/oceanprotocol/ocean-node/pull/747)
+- Add cleaner logging when idexing and getting/updating block info from db [`#743`](https://github.com/oceanprotocol/ocean-node/pull/743)
+- Add INDEXER_NETWORKS env var within CI job. [`#734`](https://github.com/oceanprotocol/ocean-node/pull/734)
+- fix system tests. Running old version of node? [`#733`](https://github.com/oceanprotocol/ocean-node/pull/733)
+- rm console.logs [`#731`](https://github.com/oceanprotocol/ocean-node/pull/731)
+- fix wrong block for log [`#727`](https://github.com/oceanprotocol/ocean-node/pull/727)
+- nonce db sql lite  [`#723`](https://github.com/oceanprotocol/ocean-node/pull/723)
+- Bump version axios 1.6.0 -> 1.7.4. [`#716`](https://github.com/oceanprotocol/ocean-node/pull/716)
+- Bump version express 4.18.2 -> 4.21.0. [`#717`](https://github.com/oceanprotocol/ocean-node/pull/717)
+- Feature/ add Elasticsearch database alternative for typesense [`#599`](https://github.com/oceanprotocol/ocean-node/pull/599)
+- Database tests 2db [`#707`](https://github.com/oceanprotocol/ocean-node/pull/707)
+- add credentials at service level, on schemas [`#710`](https://github.com/oceanprotocol/ocean-node/pull/710)
+- Bump micromatch from 4.0.5 to 4.0.8 in /dashboard [`#649`](https://github.com/oceanprotocol/ocean-node/pull/649)
+- Bump undici from 5.27.0 to 5.28.4 [`#610`](https://github.com/oceanprotocol/ocean-node/pull/610)
+- testing changes [`#718`](https://github.com/oceanprotocol/ocean-node/pull/718)
+- Policy Server  [`#694`](https://github.com/oceanprotocol/ocean-node/pull/694)
+- fix missing/invalid db_url log message, put warn at startup [`#654`](https://github.com/oceanprotocol/ocean-node/pull/654)
+- move c2d engines under OceanNode class [`#702`](https://github.com/oceanprotocol/ocean-node/pull/702)
+- improve error message, transfer fees and tweak node response [`#701`](https://github.com/oceanprotocol/ocean-node/pull/701)
+- fix default filtered p2p ip list (non routable ips that should not be announced) [`#699`](https://github.com/oceanprotocol/ocean-node/pull/699)
+- chore: add datatokenaddress in order table [`#700`](https://github.com/oceanprotocol/ocean-node/pull/700)
+- fixing tests [`#703`](https://github.com/oceanprotocol/ocean-node/pull/703)
+- Updating deprecated actions [`#681`](https://github.com/oceanprotocol/ocean-node/pull/681)
+- add access control to service level [`#698`](https://github.com/oceanprotocol/ocean-node/pull/698)
+- customize boostrap params [`#696`](https://github.com/oceanprotocol/ocean-node/pull/696)
+- Issue 668 oasis decryption [`#679`](https://github.com/oceanprotocol/ocean-node/pull/679)
+- Add max ram memory in node start cmd [`#697`](https://github.com/oceanprotocol/ocean-node/pull/697)
+- Feature/add elasticdb search [`#695`](https://github.com/oceanprotocol/ocean-node/pull/695)
+- fix: search and nonce [`#688`](https://github.com/oceanprotocol/ocean-node/pull/688)
+- Issue 501 - C2D getJobStatus - did vs documentId vs agreementId [`#502`](https://github.com/oceanprotocol/ocean-node/pull/502)
+- Feature/add elasticdb tvl [`#677`](https://github.com/oceanprotocol/ocean-node/pull/677)
+- Renaming dashboard as Control Panel [`#674`](https://github.com/oceanprotocol/ocean-node/pull/674)
+- add new schema + tests [`a648cb3`](https://github.com/oceanprotocol/ocean-node/commit/a648cb38df2dea537ca28e67ae4bf1113c9b287d)
+- normalize responses [`584b6c0`](https://github.com/oceanprotocol/ocean-node/commit/584b6c09c67fb66fee7d0d01cca51693ea2a3723)
+- first draft [`b3afc02`](https://github.com/oceanprotocol/ocean-node/commit/b3afc02219233af4026c43b886bf97c0a2edb323)
+
 #### [v0.2.0](https://github.com/oceanprotocol/ocean-node/compare/v0.1.0...v0.2.0)
 
+> 9 September 2024
+
 - Downgrade indexer errors [`#675`](https://github.com/oceanprotocol/ocean-node/pull/675)
 - Update system requirements: change iOS to macOS [`#673`](https://github.com/oceanprotocol/ocean-node/pull/673)
 - Fix crash on connection reset - P2P streams [`#659`](https://github.com/oceanprotocol/ocean-node/pull/659)

Dockerfile

@@ -4,7 +4,7 @@ COPY .nvmrc /usr/src/app/
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 ENV NVM_DIR /usr/local/nvm
 RUN mkdir $NVM_DIR
-ENV NODE_VERSION=v20.16.0
+ENV NODE_VERSION=v20.19.0
 # Install nvm with node and npm
 RUN curl https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash \
     && source $NVM_DIR/nvm.sh \
@@ -27,8 +27,8 @@ COPY . /usr/src/app
 WORKDIR /usr/src/app/
 COPY --from=builder /usr/src/app/node_modules/ /usr/src/app/node_modules/
 RUN npm run build
-# Remove the dashboard folder to reduce the image size and avoid shipping development files
-RUN rm -rf dashboard
+# Remove the controlpanel folder to reduce the image size and avoid shipping development files
+RUN rm -rf controlpanel
 ENV P2P_ipV4BindTcpPort=9000
 EXPOSE 9000
 ENV P2P_ipV4BindWsPort=9001
@@ -40,4 +40,4 @@ EXPOSE 9003
 ENV HTTP_API_PORT=8000
 EXPOSE 8000
 ENV NODE_ENV='production'
-CMD ["npm","run","start"]
+CMD ["npm","run","start"]