diff --git a/README.md b/README.md index 7602e5d..9b11091 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,5 @@ For full runbook details, please reference [the runbook.](./docs/runbook.md) ## Mercury Details This project integrates with Mercury, an indexer for Stellar/Soroban. You can find general developer documentation (in their repo docs)[https://github.com/xycloo/merury-developers-documentation/blob/main/src/SUMMARY.md]. + +For full integration details, see [the Mercury docs](./docs/mercury.md). diff --git a/docs/mercury.md b/docs/mercury.md new file mode 100644 index 0000000..9c74622 --- /dev/null +++ b/docs/mercury.md @@ -0,0 +1,64 @@ +# Mercury Integration Guide + +Freighter's backend relies on data that is ingested and indexed by Mercury in order to serve different wallet views. + +Mercury is an indexer and API built on Stellar, [see docs for more info.](https://www.mercurydata.app/) +It allows users to subscribe to data on the network and query indexed data using a GraphQL API. + +## Playground + +To learn more about the available queries or to construct queries, you can use [the playground.](https://api.mercurydata.app:2083/graphiql) + +Playground steps - + +1. Aquire an access token by signing up for an account on the dashboard. + +Testnet - https://test.mercurydata.app/ +Pubnet - https://main.mercurydata.app/ + +Select "Get Access token" under Active Subscriptions and grab your token. + +2. Add your access token to the playground. + +You can add your token to be used in the requests the playground makes. +Click on the headers tab on the left pane at the bottom and add it in the following format - + +``` +{ + Authorization: "Bearer " +} +``` + +3. Query for data in subscriptions + At this point you can query for any data that you have a subscription on, for example to get `changeTrust` operations for an account you could run this query - + +``` +query Test { + changeTrustByPublicKey(publicKeyText:"") { + edges { + node { + opId + ... + } + } + } +} +``` + +## Subscriptions + +In order to query data, you must subscribe to it first. Mercury supports subscription APIs for contract events, ledger entries, ledger entry expirations, and accounts. + +See [full docs](https://docs.mercurydata.app/mercury-classic/subscriptions/api-definition) for more info. + +## Adding a new query + +Mercury queries can be added by adding a new key to the `query` map in the [queries file](../src/service/mercury/queries.ts). +Queries are stored as template strings that represent the GraphQL query. Arguments can be passed by writing the query as a function and interpolating the arguments into the template string. + +Queries can be imported into anything that accepts GraphQL documents. + +## Usage in public facing routes + +The account history and account balance queries are used to serve public facing routes, we use [transformers](../src/service/mercury/helpers/transformers.ts) to process this data from Mercury in order to maintain schema compatibility with Horizon/RPC responses. +Any changes to these queries should be accompanied by an update to the corresponding transformer in order align with Horizon schemas used in the transformer. diff --git a/src/config.ts b/src/config.ts index 62aa085..e6adbc9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -61,6 +61,12 @@ export function buildConfig(config: Record) { config.USE_MERCURY === "true" || process.env.USE_MERCURY === "true", useSorobanPublic: true, sentryKey: config.SENTRY_KEY || process.env.SENTRY_KEY, + + blockaidConfig: { + useBlockaidDappScanning: true, + useBlockaidTxScanning: true, + useBlockaidAssetScanning: true, + }, }; } diff --git a/src/helper/error.ts b/src/helper/error.ts index 49cadc8..6d1af9e 100644 --- a/src/helper/error.ts +++ b/src/helper/error.ts @@ -22,4 +22,7 @@ export const ERROR = { UNABLE_TO_SCAN_SITE: "unable to scan site using blockaid", UNABLE_TO_SCAN_TX: "unable to scan tx using blockaid", UNABLE_TO_SCAN_ASSET: "unable to scan asset using blockaid", + SCAN_SITE_DISABLED: "scanning site using blockaid is disabled", + SCAN_TX_DISABLED: "scanning tx using blockaid is disabled", + SCAN_ASSET_DISABLED: "scanning asset using blockaid is disabled", }; diff --git a/src/helper/test-helper.ts b/src/helper/test-helper.ts index e36378f..ad7c51a 100644 --- a/src/helper/test-helper.ts +++ b/src/helper/test-helper.ts @@ -105,7 +105,7 @@ function backendClientMaker(network: NetworkNames) { } case query.getTokenBalanceSub( "CBGTG7XFRY3L6OKAUTR6KGDKUXUQBX3YDJ3QFDYTGVMOM7VV4O7NCODG", - tokenBalanceLedgerKey + tokenBalanceLedgerKey, ): { return Promise.resolve({ data: { @@ -123,7 +123,7 @@ function backendClientMaker(network: NetworkNames) { } case query.getTokenBalanceSub( "CCWAMYJME4H5CKG7OLXGC2T4M6FL52XCZ3OQOAV6LL3GLA4RO4WH3ASP", - tokenBalanceLedgerKey + tokenBalanceLedgerKey, ): { return Promise.resolve({ data: { @@ -141,7 +141,7 @@ function backendClientMaker(network: NetworkNames) { } case query.getTokenBalanceSub( "CDP3XWJ4ZN222LKYBMWIY3GYXZYX3KA6WVNDS6V7WKXSYWLAEMYW7DTZ", - tokenBalanceLedgerKey + tokenBalanceLedgerKey, ): { return Promise.resolve({ data: { @@ -160,7 +160,7 @@ function backendClientMaker(network: NetworkNames) { case query.getCurrentDataAccountBalances( pubKey, tokenBalanceLedgerKey, - [] + [], ): { return Promise.resolve({ data: queryMockResponse["query.getAccountBalancesCurrentData"], @@ -592,7 +592,7 @@ const mockMercuryClient = new MercuryClient( mercuryErrorCounter, rpcErrorCounter, criticalError, - } + }, ); jest.mock("@blockaid/client", () => { return class Blockaid {}; @@ -601,7 +601,7 @@ const blockAidClient = new Blockaid(); const blockAidService = new BlockAidService( blockAidClient, testLogger, - register + register, ); jest @@ -613,9 +613,15 @@ jest decimals: 7, symbol: "TST", }; - } + }, ); -async function getDevServer() { +async function getDevServer( + blockaidConfig = { + useBlockaidAssetScanning: true, + useBlockaidDappScanning: true, + useBlockaidTxScanning: true, + }, +) { const server = await initApiServer( mockMercuryClient, blockAidService, @@ -623,7 +629,8 @@ async function getDevServer() { true, true, register, - "development" + "development", + blockaidConfig, ); await server.listen(); diff --git a/src/index.ts b/src/index.ts index f6901a0..6b95b65 100644 --- a/src/index.ts +++ b/src/index.ts @@ -82,7 +82,7 @@ async function main() { renewClientMaker: buildRenewClientMaker(graphQlEndpoints), backendClientMaker: buildBackendClientMaker(graphQlEndpoints), currentDataClientMaker: buildCurrentDataClientMaker( - graphQlCurrentDataEndpoints + graphQlCurrentDataEndpoints, ), backends, credentials: { @@ -130,7 +130,7 @@ async function main() { rpcErrorCounter, criticalError, }, - redis + redis, ); const server = await initApiServer( mercuryClient, @@ -140,7 +140,8 @@ async function main() { conf.useSorobanPublic, register, env as mode, - redis + conf.blockaidConfig, + redis, ); const metricsServer = await initMetricsServer(register, redis); diff --git a/src/route/index.test.ts b/src/route/index.test.ts index 7db2520..6c9a19b 100644 --- a/src/route/index.test.ts +++ b/src/route/index.test.ts @@ -7,6 +7,7 @@ import { } from "../helper/test-helper"; import { transformAccountHistory } from "../service/mercury/helpers/transformers"; import { query } from "../service/mercury/queries"; +import { defaultBenignResponse } from "../service/blockaid/helpers/addScanResults"; jest.mock("@blockaid/client", () => { return class Blockaid { @@ -41,6 +42,24 @@ jest.mock("@blockaid/client", () => { return Promise.resolve({ results: res }); }, }; + token = { + scan: ({ address }: { address: string }) => { + if ( + address === + "BLND-GATALTGTWIOT6BUDBCZM3Q4OQ4BO2COLOAZ7IYSKPLC2PMSOPPGF5V56" + ) { + return Promise.resolve({ + result_type: "Malicious", + malicious_score: 1, + }); + } + + return Promise.resolve({ + result_type: "Benign", + malicious_score: 0, + }); + }, + }; }; }); @@ -288,5 +307,96 @@ describe("API routes", () => { register.clear(); await server.close(); }); + it("does not scan assets when config is disabled", async () => { + const asset_ids = [ + "BLND-GATALTGTWIOT6BUDBCZM3Q4OQ4BO2COLOAZ7IYSKPLC2PMSOPPGF5V56", + "FOO-CDP3XWJ4ZN222LKYBMWIY3GYXZYX3KA6WVNDS6V7WKXSYWLAEMYW7DTZ", + ]; + const server = await getDevServer({ + useBlockaidAssetScanning: false, + useBlockaidDappScanning: false, + useBlockaidTxScanning: false, + }); + const url = new URL( + `http://localhost:${ + (server?.server?.address() as any).port + }/api/v1/scan-asset-bulk`, + ); + url.searchParams.append("network", "PUBLIC"); + for (const id of asset_ids) { + url.searchParams.append("asset_ids", id); + } + const response = await fetch(url.href); + const data = await response.json(); + + expect(response.status).toEqual(200); + expect( + data.data.results[ + "BLND-GATALTGTWIOT6BUDBCZM3Q4OQ4BO2COLOAZ7IYSKPLC2PMSOPPGF5V56" + ], + ).toEqual({ + ...defaultBenignResponse, + }); + expect( + data.data.results[ + "FOO-CDP3XWJ4ZN222LKYBMWIY3GYXZYX3KA6WVNDS6V7WKXSYWLAEMYW7DTZ" + ], + ).toEqual({ + ...defaultBenignResponse, + }); + register.clear(); + await server.close(); + }); + }); + describe("/scan-asset", () => { + it("can scan an asset", async () => { + const server = await getDevServer(); + const url = new URL( + `http://localhost:${ + (server?.server?.address() as any).port + }/api/v1/scan-asset`, + ); + url.searchParams.append( + "address", + "BLND-GATALTGTWIOT6BUDBCZM3Q4OQ4BO2COLOAZ7IYSKPLC2PMSOPPGF5V56", + ); + const response = await fetch(url.href); + const data = await response.json(); + + expect(response.status).toEqual(200); + expect(data.data).toEqual({ + result_type: "Malicious", + malicious_score: 1, + }); + register.clear(); + await server.close(); + }); + it("does not scan an asset when config is disabled", async () => { + const server = await getDevServer({ + useBlockaidAssetScanning: false, + useBlockaidDappScanning: false, + useBlockaidTxScanning: false, + }); + const url = new URL( + `http://localhost:${ + (server?.server?.address() as any).port + }/api/v1/scan-asset`, + ); + url.searchParams.append( + "address", + "BLND-GATALTGTWIOT6BUDBCZM3Q4OQ4BO2COLOAZ7IYSKPLC2PMSOPPGF5V56", + ); + const response = await fetch(url.href); + const data = await response.json(); + + expect(response.status).toEqual(200); + expect(data.data).toEqual({ + ...defaultBenignResponse, + address: + "BLND-GATALTGTWIOT6BUDBCZM3Q4OQ4BO2COLOAZ7IYSKPLC2PMSOPPGF5V56", + }); + register.clear(); + await server.close(); + }); }); }); diff --git a/src/route/index.ts b/src/route/index.ts index 9d46088..4053efe 100644 --- a/src/route/index.ts +++ b/src/route/index.ts @@ -10,8 +10,14 @@ import { Networks } from "stellar-sdk-next"; import * as StellarSdk from "stellar-sdk"; import { MercuryClient } from "../service/mercury"; -import { BlockAidService } from "../service/blockaid"; -import { addScannedStatus } from "../service/blockaid/helpers/addScanResults"; +import { + BlockAidService, + BlockaidAssetScanResponse, +} from "../service/blockaid"; +import { + addScannedStatus, + defaultBenignResponse, +} from "../service/blockaid/helpers/addScanResults"; import { ajv } from "./validators"; import { isContractId, @@ -44,6 +50,11 @@ export async function initApiServer( useSorobanPublic: boolean, register: Prometheus.Registry, mode: mode, + blockaidConfig: { + useBlockaidDappScanning: boolean; + useBlockaidTxScanning: boolean; + useBlockaidAssetScanning: boolean; + }, redis?: Redis, ) { const routeMetricsStore = new WeakMap< @@ -323,6 +334,7 @@ export async function initApiServer( blockAidService, network, logger, + blockaidConfig.useBlockaidAssetScanning, ); } catch (e) { data.balances = data.balances.map((bal: {}) => ({ @@ -577,12 +589,20 @@ export async function initApiServer( reply, ) => { const { url } = request.query; - try { - const { data, error } = await blockAidService.scanDapp(url); - return reply.code(error ? 400 : 200).send({ data, error }); - } catch (error) { - return reply.code(500).send(ERROR.SERVER_ERROR); + if (blockaidConfig.useBlockaidDappScanning) { + try { + const { data, error } = await blockAidService.scanDapp(url); + return reply.code(error ? 400 : 200).send({ data, error }); + } catch (error) { + return reply.code(500).send(ERROR.SERVER_ERROR); + } } + return reply + .code(200) + .send({ + data: { status: "miss" }, + error: ERROR.SCAN_SITE_DISABLED, + }); }, }); @@ -618,16 +638,21 @@ export async function initApiServer( reply, ) => { const { tx_xdr, url, network } = request.query; - try { - const { data, error } = await blockAidService.scanTx( - tx_xdr, - url, - network, - ); - return reply.code(error ? 400 : 200).send({ data, error }); - } catch (error) { - return reply.code(500).send(ERROR.SERVER_ERROR); + if (blockaidConfig.useBlockaidTxScanning) { + try { + const { data, error } = await blockAidService.scanTx( + tx_xdr, + url, + network, + ); + return reply.code(error ? 400 : 200).send({ data, error }); + } catch (error) { + return reply.code(500).send(ERROR.SERVER_ERROR); + } } + return reply + .code(200) + .send({ data: null, error: ERROR.SCAN_TX_DISABLED }); }, }); @@ -654,12 +679,19 @@ export async function initApiServer( reply, ) => { const { address } = request.query; - try { - const { data, error } = await blockAidService.scanAsset(address); - return reply.code(error ? 400 : 200).send({ data, error }); - } catch (error) { - return reply.code(500).send(ERROR.SERVER_ERROR); + + if (blockaidConfig.useBlockaidAssetScanning) { + try { + const { data, error } = await blockAidService.scanAsset(address); + return reply.code(error ? 400 : 200).send({ data, error }); + } catch (error) { + return reply.code(500).send(ERROR.SERVER_ERROR); + } } + return reply.code(200).send({ + data: { ...defaultBenignResponse, address }, + error: ERROR.SCAN_ASSET_DISABLED, + }); }, }); @@ -690,13 +722,27 @@ export async function initApiServer( reply, ) => { const { asset_ids } = request.query; - try { - const { data, error } = - await blockAidService.scanAssetBulk(asset_ids); - return reply.code(error ? 400 : 200).send({ data, error }); - } catch (error) { - return reply.code(500).send(ERROR.SERVER_ERROR); + if (blockaidConfig.useBlockaidAssetScanning) { + try { + const { data, error } = + await blockAidService.scanAssetBulk(asset_ids); + return reply.code(error ? 400 : 200).send({ data, error }); + } catch (error) { + return reply.code(500).send(ERROR.SERVER_ERROR); + } } + const defaultResponse: { + [addres: string]: BlockaidAssetScanResponse; + } = {}; + asset_ids.forEach((address) => { + defaultResponse[address] = { + ...defaultBenignResponse, + }; + }); + return reply.code(200).send({ + data: { results: defaultResponse }, + error: ERROR.SCAN_ASSET_DISABLED, + }); }, }); diff --git a/src/service/blockaid/helpers/addScanResults.ts b/src/service/blockaid/helpers/addScanResults.ts index 28d6429..3861928 100644 --- a/src/service/blockaid/helpers/addScanResults.ts +++ b/src/service/blockaid/helpers/addScanResults.ts @@ -3,11 +3,27 @@ import { Logger } from "pino"; import { BlockAidService } from ".."; import { NetworkNames } from "../../../helper/validate"; +export const defaultBenignResponse: Blockaid.Token.TokenScanResponse = { + result_type: "Benign", + malicious_score: "0.0", + attack_types: {}, + chain: "stellar", + address: "", + metadata: { + type: "", + }, + fees: {}, + features: [], + trading_limits: {}, + financial_stats: {}, +}; + export const addScannedStatus = async ( balances: { [key: string]: {} }, blockaidService: BlockAidService, network: NetworkNames, logger: Logger, + useBlockaidAssetScanning: boolean, ) => { const scannedBalances = {} as { [key: string]: { blockaidData: Blockaid.Token.TokenScanResponse }; @@ -33,23 +49,12 @@ export const addScannedStatus = async ( scannedBalances[key] = { ...balanceInfo, blockaidData: { - result_type: "Benign", - malicious_score: "0.0", - attack_types: {}, - chain: "stellar", - address: "", - metadata: { - type: "", - }, - fees: {}, - features: [], - trading_limits: {}, - financial_stats: {}, + ...defaultBenignResponse, }, }; } - if (network === "PUBLIC") { + if (network === "PUBLIC" && useBlockaidAssetScanning) { // we only scan non-native assets on the public network try { const bulkRes = await blockaidService.scanAssetBulk(keyList); diff --git a/src/service/blockaid/index.ts b/src/service/blockaid/index.ts index a2bd26c..526b83f 100644 --- a/src/service/blockaid/index.ts +++ b/src/service/blockaid/index.ts @@ -3,6 +3,7 @@ import Prometheus from "prom-client"; import { Logger } from "pino"; import { Networks, TransactionBuilder } from "stellar-sdk"; +import { defaultBenignResponse } from "./helpers/addScanResults"; import { ERROR } from "../../helper/error"; import { NetworkNames } from "../../helper/validate"; @@ -14,6 +15,8 @@ const NetworkNameBlockaid: { TESTNET: "testnet", }; +export type BlockaidAssetScanResponse = Blockaid.Token.TokenScanResponse; + export class BlockAidService { blockAidClient: Blockaid; logger: Logger; @@ -45,7 +48,7 @@ export class BlockAidService { } catch (error) { this.logger.error(error); this.scanMissCounter.inc(); - return { data: null, error: ERROR.UNABLE_TO_SCAN_SITE }; + return { data: { status: "miss" }, error: ERROR.UNABLE_TO_SCAN_SITE }; } }; @@ -113,7 +116,19 @@ export class BlockAidService { } catch (error) { this.logger.error(error); this.scanMissCounter.inc(); - return { data: null, error: ERROR.UNABLE_TO_SCAN_ASSET }; + const defaultResponse: { + [addres: string]: Blockaid.Token.TokenScanResponse; + } = {}; + addressList.forEach((address) => { + defaultResponse[address] = { + ...defaultBenignResponse, + }; + }); + + return { + data: { results: defaultResponse }, + error: ERROR.UNABLE_TO_SCAN_ASSET, + }; } }; } diff --git a/yarn.lock b/yarn.lock index 2fdc80f..f042549 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2218,9 +2218,9 @@ fill-range@^7.1.1: to-regex-range "^5.0.1" find-my-way@^8.0.0: - version "8.2.0" - resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.2.0.tgz#ef1b83d008114a300118c9c707d8dc65947d9960" - integrity sha512-HdWXgFYc6b1BJcOBDBwjqWuHJj1WYiqrxSh25qtU4DabpMFdj/gSunNBQb83t+8Zt67D7CXEzJWTkxaShMTMOA== + version "8.2.2" + resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-8.2.2.tgz#f3e78bc6ead2da4fdaa201335da3228600ed0285" + integrity sha512-Dobi7gcTEq8yszimcfp/R7+owiT4WncAJ7VTTgFH1jYJ5GaG1FbhjwDG820hptN0QDFvzVY3RfCzdInvGPGzjA== dependencies: fast-deep-equal "^3.1.3" fast-querystring "^1.0.0"