refactor: remove old token metadata processor (#1747)

* refactor: remove old processor, start fetching data from contract

* refactor: remove old tests

* fix: tests and lint

* fix: use lru cache

* chore: delete old tables

* chore: fix comments [skip ci]

* fix: env comment block [skip ci]

* chore: avoid duplicate RPC lookups

* fix: make an incremental migration to drop tables

* fix: re-add metadata processing ENV flag

---------

Co-authored-by: Matthew Little <[email protected]>

Author: rafaelcr

.env

@@ -122,34 +122,17 @@ STACKS_NODE_TYPE=L1
 # Override the default file path for the proxy cache control file
 # STACKS_API_PROXY_CACHE_CONTROL_FILE=/path/to/.proxy-cache-control.json
 
-# Enable token metadata processing. Disabled by default.
+# Enable FT metadata processing for Rosetta operations display. Disabled by default.
 # STACKS_API_ENABLE_FT_METADATA=1
-# STACKS_API_ENABLE_NFT_METADATA=1
-
-# If token metadata processing is enabled, this variable determines how the API reacts to metadata processing failures.
-# When strict mode is enabled, any failures caused by recoverable errors will be retried indefinitely. Otherwise,
-# the API will give up after `STACKS_API_TOKEN_METADATA_MAX_RETRIES` is reached for that smart contract.
-# STACKS_API_TOKEN_METADATA_STRICT_MODE=1
-
-# Maximum number of times we'll try processing FT/NFT metadata for a specific smart contract if we've failed
-# because of a recoverable error.
-# Only used if `STACKS_API_TOKEN_METADATA_STRICT_MODE` is disabled.
-# STACKS_API_TOKEN_METADATA_MAX_RETRIES=5
-
-# Controls the token metadata error handling mode. The possible values are:
-# * `warning`: If required metadata is not found, the API will issue a warning and not display data for that token.
-# * `error`: If required metadata is not found, the API will throw an error.
-# If not specified or any other value is provided, the mode will be set to `warning`.
-# STACKS_API_TOKEN_METADATA_ERROR_MODE=warning
 
-# Configure a script to handle image URLs during token metadata processing.
-# This example script uses the `imgix.net` service to create CDN URLs.
-# Must be an executable script that accepts the URL as the first program argument
-# and outputs a result URL to stdout.
-# STACKS_API_IMAGE_CACHE_PROCESSOR=./config/token-metadata-image-cache-imgix.js
-# Env vars needed for the above sample `imgix` script:
-# IMGIX_DOMAIN=https://<your domain>.imgix.net
-# IMGIX_TOKEN=<your token>
+# The Rosetta API endpoints require FT metadata to display operations with the proper `symbol` and
+# `decimals` values. If FT metadata is enabled, this variable controls the token metadata error
+# handling mode when metadata is not found.
+# The possible values are:
+# * `warning`: The API will issue a warning and not display data for that token.
+# * `error`: The API will throw an error. If not specified or any other value is provided, the mode
+#            will be set to `warning`.
+# STACKS_API_TOKEN_METADATA_ERROR_MODE=warning
 
 # Web Socket ping interval to determine client availability, in seconds.
 # STACKS_API_WS_PING_INTERVAL=5

.github/workflows/ci.yml

@@ -848,96 +848,6 @@ jobs:
           flag-name: run-${{ github.job }}
           parallel: true
 
-  test-tokens-metadata:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Use Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-
-      - name: Install deps
-        run: npm ci
-
-      - name: Setup env vars
-        run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0" >> $GITHUB_ENV
-
-      - name: Setup integration environment
-        run: |
-          sudo ufw disable
-          npm run devenv:deploy-krypton -- -d
-          npm run devenv:logs-krypton -- --no-color &> docker-compose-logs.txt &
-
-      - name: Run tokens tests
-        run: npm run test:tokens-metadata
-
-      - name: Print integration environment logs
-        run: cat docker-compose-logs.txt
-        if: failure()
-
-      - name: Teardown integration environment
-        run: npm run devenv:stop-krypton
-        if: always()
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        if: always()
-
-      - name: Upload coverage to Coveralls
-        uses: coverallsapp/github-action@master
-        if: ${{ false }}
-        with:
-          github-token: ${{ secrets.github_token }}
-          flag-name: run-${{ github.job }}
-          parallel: true
-
-  test-tokens-strict:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Use Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-
-      - name: Install deps
-        run: npm ci
-
-      - name: Setup env vars
-        run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0" >> $GITHUB_ENV
-
-      - name: Setup integration environment
-        run: |
-          sudo ufw disable
-          npm run devenv:deploy-krypton -- -d
-          npm run devenv:logs-krypton -- --no-color &> docker-compose-logs.txt &
-
-      - name: Run tokens tests
-        run: npm run test:tokens-strict
-
-      - name: Print integration environment logs
-        run: cat docker-compose-logs.txt
-        if: failure()
-
-      - name: Teardown integration environment
-        run: npm run devenv:stop-krypton
-        if: always()
-
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        if: always()
-
-      - name: Upload coverage to Coveralls
-        uses: coverallsapp/github-action@master
-        if: ${{ false }}
-        with:
-          github-token: ${{ secrets.github_token }}
-          flag-name: run-${{ github.job }}
-          parallel: true
-
   build-publish:
     runs-on: ubuntu-latest
     needs:
@@ -948,7 +858,6 @@ jobs:
       - test-bns
       - test-rosetta
       - test-rosetta-cli-construction
-      - test-tokens-strict
     steps:
       - uses: actions/checkout@v3
         with:

.vscode/launch.json

@@ -346,40 +346,6 @@
       "preLaunchTask": "deploy:krypton",
       "postDebugTask": "stop:krypton",
     },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Jest: Tokens - strict mode",
-      "program": "${workspaceFolder}/node_modules/.bin/jest",
-      "args": [
-        "--testTimeout=3600000",
-        "--runInBand",
-        "--no-cache",
-        "--config",
-        "${workspaceRoot}/tests/jest.config.tokens-strict.js",
-      ],
-      "outputCapture": "std",
-      "console": "integratedTerminal",
-      "preLaunchTask": "deploy:krypton",
-      "postDebugTask": "stop:krypton",
-    },
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Jest: Tokens - metadata",
-      "program": "${workspaceFolder}/node_modules/.bin/jest",
-      "args": [
-        "--testTimeout=3600000",
-        "--runInBand",
-        "--no-cache",
-        "--config",
-        "${workspaceRoot}/tests/jest.config.tokens-metadata.js",
-      ],
-      "outputCapture": "std",
-      "console": "integratedTerminal",
-      "preLaunchTask": "deploy:krypton",
-      "postDebugTask": "stop:krypton",
-    },
     {
       "type": "node",
       "request": "launch",

migrations/1699540187362_remove-token-metadata.js

@@ -0,0 +1,132 @@
+/* eslint-disable camelcase */
+
+exports.shorthands = undefined;
+
+exports.up = pgm => {
+  pgm.dropTable('token_metadata_queue');
+  pgm.dropTable('nft_metadata');
+  pgm.dropTable('ft_metadata');
+};
+
+exports.down = pgm => {
+  pgm.createTable('token_metadata_queue', {
+    queue_id: {
+      type: 'serial',
+      primaryKey: true,
+    },
+    tx_id: {
+      type: 'bytea',
+      notNull: true,
+    },
+    contract_id: {
+      type: 'string',
+      notNull: true,
+    },
+    contract_abi: {
+      type: 'string',
+      notNull: true,
+    },
+    block_height: {
+      type: 'integer',
+      notNull: true,
+    },
+    processed: {
+      type: 'boolean',
+      notNull: true,
+    },
+    retry_count: {
+      type: 'integer',
+      notNull: true,
+      default: 0,
+    }
+  });
+  pgm.createIndex('token_metadata_queue', [{ name: 'block_height', sort: 'DESC' }]);
+  pgm.createTable('nft_metadata', {
+    id: {
+      type: 'serial',
+      primaryKey: true,
+    },
+    name: {
+      type: 'string',
+      notNull: true,
+    },
+    token_uri: {
+      type: 'string',
+      notNull: true,
+    },
+    description: {
+      type: 'string',
+      notNull: true,
+    },
+    image_uri: {
+      type: 'string',
+      notNull: true,
+    },
+    image_canonical_uri: {
+      type: 'string',
+      notNull: true,
+    },
+    contract_id: {
+      type: 'string',
+      notNull: true,
+      unique: true,
+    },
+    tx_id: {
+      type: 'bytea',
+      notNull: true,
+    },
+    sender_address: {
+      type: 'string',
+      notNull: true,
+    }
+  });
+  pgm.createIndex('nft_metadata', 'contract_id', { method: 'hash' });
+  pgm.createTable('ft_metadata', {
+    id: {
+      type: 'serial',
+      primaryKey: true,
+    },
+    name: {
+      type: 'string',
+      notNull: true,
+    },
+    token_uri: {
+      type: 'string',
+      notNull: true,
+    },
+    description: {
+      type: 'string',
+      notNull: true,
+    },
+    image_uri: {
+      type: 'string',
+      notNull: true,
+    },
+    image_canonical_uri: {
+      type: 'string',
+      notNull: true,
+    },
+    contract_id: {
+      type: 'string',
+      notNull: true,
+      unique: true,
+    },
+    symbol: {
+      type: 'string',
+      notNull: true,
+    },
+    decimals: {
+      type: 'integer',
+      notNull: true,
+    },
+    tx_id: {
+      type: 'bytea',
+      notNull: true,
+    },
+    sender_address: {
+      type: 'string',
+      notNull: true,
+    }
+  });
+  pgm.createIndex('ft_metadata', 'contract_id', { method: 'hash' });
+}

package.json

@@ -31,8 +31,6 @@
     "test:integration:rosetta-cli:construction": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test STACKS_CHAIN_ID=0x80000000 jest --config ./tests/jest.config.rosetta-cli-construction.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
     "test:integration:bns": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.bns.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
     "test:integration:bns-e2e": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.bns-e2e.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
-    "test:integration:tokens-strict": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.tokens-strict.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
-    "test:integration:tokens-metadata": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.tokens-metadata.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
     "test:integration:rpc": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.rpc.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
     "test:integration:event-replay": "concurrently \"docker compose -f docker/docker-compose.dev.postgres.yml up --force-recreate -V\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.event-replay.js --no-cache --runInBand; npm run devenv:stop\"",
     "test:integration:btc-faucet": "concurrently \"docker compose -f docker/docker-compose.dev.postgres.yml -f docker/docker-compose.dev.bitcoind.yml up --force-recreate -V\" \"cross-env NODE_ENV=test jest --config ./tests/jest.config.btc-faucet.js --no-cache --runInBand; npm run devenv:stop\"",

src/api/controllers/db-controller.ts

@@ -63,7 +63,7 @@ import {
 } from '../../datastore/common';
 import { unwrapOptional, FoundOrNot, unixEpochToIso, EMPTY_HASH_256, ChainID } from '../../helpers';
 import { serializePostCondition, serializePostConditionMode } from '../serializers/post-conditions';
-import { getOperations, parseTransactionMemo } from '../../rosetta-helpers';
+import { getOperations, parseTransactionMemo } from '../../rosetta/rosetta-helpers';
 import { PgStore } from '../../datastore/pg-store';
 import { Pox2EventName } from '../../pox-helpers';
 import { logger } from '../../logger';
@@ -164,30 +164,6 @@ export function getTxStatus(txStatus: DbTxStatus | string): string {
   }
 }
 
-type EventTypeString =
-  | 'smart_contract_log'
-  | 'stx_asset'
-  | 'fungible_token_asset'
-  | 'non_fungible_token_asset'
-  | 'stx_lock';
-
-export function getEventTypeString(eventTypeId: DbEventTypeId): EventTypeString {
-  switch (eventTypeId) {
-    case DbEventTypeId.SmartContractLog:
-      return 'smart_contract_log';
-    case DbEventTypeId.StxAsset:
-      return 'stx_asset';
-    case DbEventTypeId.FungibleTokenAsset:
-      return 'fungible_token_asset';
-    case DbEventTypeId.NonFungibleTokenAsset:
-      return 'non_fungible_token_asset';
-    case DbEventTypeId.StxLock:
-      return 'stx_lock';
-    default:
-      throw new Error(`Unexpected DbEventTypeId: ${eventTypeId}`);
-  }
-}
-
 export function getAssetEventTypeString(
   assetEventTypeId: DbAssetEventTypeId
 ): 'transfer' | 'mint' | 'burn' {

src/api/init.ts

@@ -31,7 +31,7 @@ import * as expressListEndpoints from 'express-list-endpoints';
 import { createMiddleware as createPrometheusMiddleware } from '@promster/express';
 import { createMicroblockRouter } from './routes/microblock';
 import { createStatusRouter } from './routes/status';
-import { createTokenRouter } from './routes/tokens/tokens';
+import { createTokenRouter } from './routes/tokens';
 import { createFeeRateRouter } from './routes/fee-rate';
 import { setResponseNonCacheable } from './controllers/cache-controller';