diff --git a/apps/price_pusher/README.md b/apps/price_pusher/README.md index 882b6a872f..f55bb49b66 100644 --- a/apps/price_pusher/README.md +++ b/apps/price_pusher/README.md @@ -159,7 +159,7 @@ pnpm run start solana \ --endpoint https://api.mainnet-beta.solana.com \ --keypair-file ./id.json \ --shard-id 1 \ - --jito-endpoint mainnet.block-engine.jito.wtf \ + --jito-endpoints mainnet.block-engine.jito.wtf,ny.mainnet.block-engine.jito.wtf \ --jito-keypair-file ./jito.json \ --jito-tip-lamports 100000 \ --jito-bundle-size 5 \ diff --git a/apps/price_pusher/src/solana/command.ts b/apps/price_pusher/src/solana/command.ts index 93420ef79d..8f82aad30d 100644 --- a/apps/price_pusher/src/solana/command.ts +++ b/apps/price_pusher/src/solana/command.ts @@ -49,8 +49,8 @@ export default { type: "number", default: 50000, } as Options, - "jito-endpoint": { - description: "Jito endpoint", + "jito-endpoints": { + description: "Jito endpoint(s) - comma-separated list of endpoints", type: "string", optional: true, } as Options, @@ -117,7 +117,7 @@ export default { pythContractAddress, pushingFrequency, pollingFrequency, - jitoEndpoint, + jitoEndpoints, jitoKeypairFile, jitoTipLamports, dynamicJitoTips, @@ -209,7 +209,18 @@ export default { Uint8Array.from(JSON.parse(fs.readFileSync(jitoKeypairFile, "ascii"))), ); - const jitoClient = searcherClient(jitoEndpoint, jitoKeypair); + const jitoEndpointsList = jitoEndpoints + .split(",") + .map((endpoint: string) => endpoint.trim()); + const jitoClients: SearcherClient[] = jitoEndpointsList.map( + (endpoint: string) => { + logger.info( + `Constructing Jito searcher client from endpoint ${endpoint}`, + ); + return searcherClient(endpoint, jitoKeypair); + }, + ); + solanaPricePusher = new SolanaPricePusherJito( pythSolanaReceiver, hermesClient, @@ -218,13 +229,16 @@ export default { jitoTipLamports, dynamicJitoTips, maxJitoTipLamports, - jitoClient, + jitoClients, jitoBundleSize, updatesPerJitoBundle, + 60000, // Default max retry time of 60 seconds lookupTableAccount, ); - onBundleResult(jitoClient, logger.child({ module: "JitoClient" })); + jitoClients.forEach((client, index) => { + onBundleResult(client, logger.child({ module: `JitoClient-${index}` })); + }); } else { solanaPricePusher = new SolanaPricePusher( pythSolanaReceiver, diff --git a/apps/price_pusher/src/solana/solana.ts b/apps/price_pusher/src/solana/solana.ts index 1213490cc2..ccfeaaea3f 100644 --- a/apps/price_pusher/src/solana/solana.ts +++ b/apps/price_pusher/src/solana/solana.ts @@ -166,9 +166,10 @@ export class SolanaPricePusherJito implements IPricePusher { private defaultJitoTipLamports: number, private dynamicJitoTips: boolean, private maxJitoTipLamports: number, - private searcherClient: SearcherClient, + private searcherClients: SearcherClient[], private jitoBundleSize: number, private updatesPerJitoBundle: number, + private maxRetryTimeMs: number = 60000, // Default to 60 seconds max retry time private addressLookupTableAccount?: AddressLookupTableAccount, ) {} @@ -243,27 +244,36 @@ export class SolanaPricePusherJito implements IPricePusher { jitoBundleSize: this.jitoBundleSize, }); - let retries = 60; - while (retries > 0) { - try { - await sendTransactionsJito( - transactions, - this.searcherClient, - this.pythSolanaReceiver.wallet, - ); - break; - } catch (err: any) { - if (err.code === 8 && err.details?.includes("Rate limit exceeded")) { - this.logger.warn("Rate limit hit, waiting before retry..."); - await this.sleep(1100); // Wait slightly more than 1 second - retries--; - if (retries === 0) { - this.logger.error("Max retries reached for rate limit"); - throw err; - } - } else { - throw err; + try { + this.logger.info("Sending Jito transactions..."); + await sendTransactionsJito( + transactions, + this.searcherClients, + this.pythSolanaReceiver.wallet, + { maxRetryTimeMs: this.maxRetryTimeMs }, + ); + } catch (err: any) { + if (err.code === 8 && err.details?.includes("Rate limit exceeded")) { + this.logger.warn("Rate limit hit, waiting before retry..."); + await this.sleep(1100); // Wait slightly more than 1 second + try { + this.logger.info("Sending Jito transactions..."); + await sendTransactionsJito( + transactions, + this.searcherClients, + this.pythSolanaReceiver.wallet, + { maxRetryTimeMs: this.maxRetryTimeMs }, + ); + } catch (retryErr: any) { + this.logger.error("Failed after rate limit retry"); + throw retryErr; } + } else { + this.logger.error( + { err }, + "Failed to send transactions via all JITO endpoints", + ); + throw err; } } diff --git a/target_chains/solana/sdk/js/solana_utils/package.json b/target_chains/solana/sdk/js/solana_utils/package.json index d647e92b53..20fea33bb9 100644 --- a/target_chains/solana/sdk/js/solana_utils/package.json +++ b/target_chains/solana/sdk/js/solana_utils/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/solana-utils", - "version": "0.4.4", + "version": "0.4.5", "description": "Utility functions for Solana", "homepage": "https://pyth.network", "main": "lib/index.js", diff --git a/target_chains/solana/sdk/js/solana_utils/src/jito.ts b/target_chains/solana/sdk/js/solana_utils/src/jito.ts index 2a6a5c290e..5db14c1315 100644 --- a/target_chains/solana/sdk/js/solana_utils/src/jito.ts +++ b/target_chains/solana/sdk/js/solana_utils/src/jito.ts @@ -42,9 +42,23 @@ export async function sendTransactionsJito( tx: VersionedTransaction; signers?: Signer[] | undefined; }[], - searcherClient: SearcherClient, + searcherClients: SearcherClient | SearcherClient[], wallet: Wallet, + options: { + maxRetryTimeMs?: number; + } = {}, ): Promise { + const clients = Array.isArray(searcherClients) + ? searcherClients + : [searcherClients]; + + if (clients.length === 0) { + throw new Error("No searcher clients provided"); + } + + const maxRetryTimeMs = options.maxRetryTimeMs || 60000; // Default to 60 seconds + const startTime = Date.now(); + const signedTransactions = []; for (const transaction of transactions) { @@ -64,7 +78,24 @@ export async function sendTransactionsJito( ); const bundle = new Bundle(signedTransactions, 2); - await searcherClient.sendBundle(bundle); - return firstTransactionSignature; + let lastError: Error | null = null; + let clientIndex = 0; + + while (Date.now() - startTime < maxRetryTimeMs) { + const currentClient = clients[clientIndex]; + try { + await currentClient.sendBundle(bundle); + return firstTransactionSignature; + } catch (err: any) { + lastError = err; + clientIndex = (clientIndex + 1) % clients.length; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } + + throw ( + lastError || + new Error("Failed to send transactions via JITO after maximum retry time") + ); }