feat(backend): multi-hop payments with static routing

localenv/admin-auth/docker-compose.yml

@@ -27,8 +27,10 @@ services:
       args:
         PATH_TO_KRATOS_CONFIG: ./localenv/admin-auth/cloud-nine-kratos.yml
     depends_on:
-      - shared-database
-      - mailslurper
+      shared-database:
+        condition: service_healthy
+      mailslurper:
+        condition: service_started
     environment:
       DEV_MODE: true
     ports:
@@ -43,8 +45,10 @@ services:
       args:
         PATH_TO_KRATOS_CONFIG: ./localenv/admin-auth/happy-life-kratos.yml
     depends_on:
-      - shared-database
-      - mailslurper
+      shared-database:
+        condition: service_healthy
+      mailslurper:
+        condition: service_started
     environment:
       DEV_MODE: true
     ports:

localenv/cloud-nine-wallet/docker-compose.yml

@@ -33,6 +33,8 @@ services:
     depends_on:
       cloud-nine-backend:
         condition: service_healthy
+      shared-database:
+        condition: service_healthy
   cloud-nine-backend:
     hostname: cloud-nine-wallet-backend
     image: rafiki-backend
@@ -84,8 +86,10 @@ services:
       KEY_ID: 7097F83B-CB84-469E-96C6-2141C72E22C0
       OPERATOR_TENANT_ID: 438fa74a-fa7d-4317-9ced-dde32ece1787
     depends_on:
-      - shared-database
-      - shared-redis
+      shared-database:
+        condition: service_healthy
+      shared-redis:
+        condition: service_started
     healthcheck:
       test: ["CMD", "wget", "--spider", "http://localhost:3001/healthz"]
       start_period: 60s
@@ -126,8 +130,10 @@ services:
       OPERATOR_TENANT_ID: 438fa74a-fa7d-4317-9ced-dde32ece1787
       SERVICE_API_PORT: 3011
     depends_on:
-      - shared-database
-      - shared-redis
+      shared-database:
+        condition: service_healthy
+      shared-redis:
+        condition: service_started
   shared-database:
     image: 'postgres:15' # use latest official postgres version
     restart: unless-stopped
@@ -141,6 +147,12 @@ services:
     environment:
       POSTGRES_PASSWORD: password
       POSTGRES_USER: postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
   shared-redis:
     image: 'redis:7'
     restart: unless-stopped

localenv/global-bank/docker-compose.yml

@@ -29,12 +29,8 @@ services:
     depends_on:
       global-bank-backend:
         condition: service_healthy
-      intergalactic-bank-mock-ase:
-        condition: service_started
-      cloud-nine-mock-ase:
-        condition: service_started
-      cloud-ten-mock-ase:
-        condition: service_started
+      shared-database:
+        condition: service_healthy
   global-bank-backend:
     hostname: global-bank-backend
     image: rafiki-backend
@@ -81,7 +77,8 @@ services:
       KEY_ID: 53f2d913-e98a-40b9-b270-372d0547f23e
       OPERATOR_TENANT_ID: 53f2d913-e98a-40b9-b270-372d0547f23e
     depends_on:
-      - intergalactic-bank-backend
+      shared-database:
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "wget", "--spider", "http://localhost:3001/healthz"]
       start_period: 60s

localenv/intergalactic-bank/docker-compose.yml

@@ -33,6 +33,8 @@ services:
     depends_on:
       intergalactic-bank-backend:
         condition: service_healthy
+      shared-database:
+        condition: service_healthy
 
   intergalactic-bank-backend:
     hostname: intergalactic-bank-backend
@@ -83,8 +85,10 @@ services:
       OPERATOR_TENANT_ID: 8A1B2C3D-4E5F-6789-ABCD-1234567890AB
       ENABLE_ILP_TIMING: true
     depends_on:
-      - shared-database
-      - shared-redis
+      shared-database:
+        condition: service_healthy
+      shared-redis:
+        condition: service_started
     healthcheck:
       test: ["CMD", "wget", "--spider", "http://localhost:3001/healthz"]
       start_period: 60s
@@ -126,8 +130,10 @@ services:
       OPERATOR_TENANT_ID: 8A1B2C3D-4E5F-6789-ABCD-1234567890AB
       SERVICE_API_PORT: 3011
     depends_on:
-      - shared-database
-      - shared-redis
+      shared-database:
+        condition: service_healthy
+      shared-redis:
+        condition: service_started
 
   intergalactic-bank-admin:
     hostname: intergalactic-bank-admin

localenv/us-treasury/docker-compose.yml

@@ -33,6 +33,8 @@ services:
     depends_on:
       us-treasury-backend:
         condition: service_healthy
+      shared-database:
+        condition: service_healthy
 
   us-treasury-backend:
     hostname: us-treasury-backend
@@ -83,8 +85,10 @@ services:
       KEY_ID: 7097F83B-CB84-469E-96C6-2141C72E22C0
       OPERATOR_TENANT_ID: 7097F83B-CB84-469E-96C6-2141C72E22C0
     depends_on:
-      - shared-database
-      - shared-redis
+      shared-database:
+        condition: service_healthy
+      shared-redis:
+        condition: service_started
     healthcheck:
       test: ["CMD", "wget", "--spider", "http://localhost:3001/healthz"]
       start_period: 60s
@@ -126,8 +130,10 @@ services:
       OPERATOR_TENANT_ID: 7097F83B-CB84-469E-96C6-2141C72E22C0
       SERVICE_API_PORT: 3011
     depends_on:
-      - shared-database
-      - shared-redis
+      shared-database:
+        condition: service_healthy
+      shared-redis:
+        condition: service_started
 
   us-treasury-admin:
     hostname: us-treasury-admin

packages/backend/src/accounting/service.ts

@@ -155,6 +155,76 @@ interface CreateAccountToAccountTransferArgs {
   ): Promise<string[] | TransferError>
 }
 
+// Helper function to identify if an account is a peer account
+function isPeerAccount(account: LiquidityAccount): boolean {
+  return (
+    account.onDebit !== undefined && 
+    account.id !== account.asset.id &&
+    // Check for peer-specific properties
+    (account as any).staticIlpAddress !== undefined
+  )
+}
+
+// Helper function to log peer liquidity before and after transfers
+async function logPeerLiquidity(
+  deps: BaseAccountingServiceDependencies,
+  accounts: LiquidityAccount[],
+  getAccountBalance: (accountRef: string) => Promise<bigint | undefined>,
+  stage: 'before' | 'after',
+  transferInfo?: { sourceAmount: bigint; destinationAmount?: bigint }
+): Promise<void> {
+  const peerAccounts = accounts.filter(isPeerAccount)
+
+  if (peerAccounts.length === 0) {
+    return
+  }
+
+  const peerBalances = await Promise.all(
+    peerAccounts.map(async (account) => {
+      const balance = await getAccountBalance(account.id)
+      const peer = account as any
+      return {
+        peerId: account.id,
+        staticIlpAddress: peer.staticIlpAddress || 'Unknown',
+        balance: balance || 0n,
+        balanceFormatted: formatBalance(balance || 0n, account.asset.scale || 0),
+        assetCode: account.asset.code || 'Unknown',
+        assetScale: account.asset.scale || 0,
+        liquidityThreshold: peer.liquidityThreshold || null
+      }
+    })
+  )
+
+  const logData = {
+    stage,
+    peerBalances,
+    transferInfo: transferInfo ? {
+      sourceAmount: transferInfo.sourceAmount.toString(),
+      sourceAmountFormatted: formatBalance(transferInfo.sourceAmount, accounts[0]?.asset.scale || 0),
+      destinationAmount: transferInfo.destinationAmount?.toString(),
+      destinationAmountFormatted: transferInfo.destinationAmount ? 
+        formatBalance(transferInfo.destinationAmount, accounts[1]?.asset.scale || 0) : undefined
+    } : undefined
+  }
+
+  deps.logger.info(logData, `Peer liquidity ${stage} payment transfer`)
+}
+
+function formatBalance(balance: bigint, scale: number): string {
+  const balanceStr = balance.toString()
+  if (scale === 0) {
+    return balanceStr
+  }
+
+  if (balanceStr.length <= scale) {
+    return `0.${balanceStr.padStart(scale, '0')}`
+  }
+
+  const integerPart = balanceStr.slice(0, -scale)
+  const decimalPart = balanceStr.slice(-scale)
+  return `${integerPart}.${decimalPart}`
+}
+
 export async function createAccountToAccountTransfer(
   deps: BaseAccountingServiceDependencies,
   args: CreateAccountToAccountTransferArgs
@@ -185,6 +255,14 @@ export async function createAccountToAccountTransfer(
     return TransferError.InvalidDestinationAmount
   }
 
+  await logPeerLiquidity(
+    deps,
+    [sourceAccount, destinationAccount],
+    getAccountBalance,
+    'before',
+    { sourceAmount, destinationAmount }
+  )
+
   const transfersToCreateOrError =
     sourceAccount.asset.ledger === destinationAccount.asset.ledger
       ? buildSameAssetTransfers(transferArgs)
@@ -241,6 +319,14 @@ export async function createAccountToAccountTransfer(
           withdrawalThrottleDelay
         })
       }
+
+      await logPeerLiquidity(
+        deps,
+        [sourceAccount, destinationAccount],
+        getAccountBalance,
+        'after',
+        { sourceAmount, destinationAmount }
+      )
     },
 
     void: async (): Promise<void | TransferError> => {

packages/backend/src/payment-method/ilp/connector/core/middleware/account.ts

@@ -1,7 +1,7 @@
 import { Errors } from 'ilp-packet'
 import { AccountAlreadyExistsError } from '../../../../../accounting/errors'
 import { LiquidityAccountType } from '../../../../../accounting/service'
-import { IncomingPaymentState } from '../../../../../open_payments/payment/incoming/model'
+import { IncomingPayment, IncomingPaymentState } from '../../../../../open_payments/payment/incoming/model'
 import {
   ILPContext,
   ILPMiddleware,
@@ -10,10 +10,30 @@ import {
 } from '../rafiki'
 import { AuthState } from './auth'
 import { StreamState } from './stream-address'
+import { OutgoingPayment } from '../../../../../open_payments/payment/outgoing/model'
+import { Peer } from '../../../peer/model'
+
+function determineOperation(
+  incoming: IncomingAccount,
+  outgoing: OutgoingAccount
+): string {
+  if (incoming instanceof OutgoingPayment && outgoing instanceof Peer) {
+    return 'outgoing_payment'
+  }
+
+  if (incoming instanceof Peer && outgoing instanceof IncomingPayment) {
+    return 'incoming_payment'
+  }
+
+  if (incoming instanceof Peer && outgoing instanceof Peer) {
+    return 'routing'
+  }
+  return 'unknown'
+}
 
 export function createAccountMiddleware(): ILPMiddleware {
   return async function account(
-    ctx: ILPContext<AuthState & StreamState>,
+    ctx: ILPContext<AuthState & StreamState & { unfulfillable: boolean, operation: string }>,
     next: () => Promise<void>
   ): Promise<void> {
     const createLiquidityAccount = async (
@@ -105,7 +125,8 @@ export function createAccountMiddleware(): ILPMiddleware {
       const address = ctx.request.prepare.destination
       const peer = await peers.getByDestinationAddress(
         address,
-        incomingAccount.tenantId
+        incomingAccount.tenantId,
+        incomingAccount.id
       )
       if (peer) {
         return peer
@@ -134,6 +155,15 @@ export function createAccountMiddleware(): ILPMiddleware {
         return outgoingAccount!
       }
     }
+
+    const operation = determineOperation(incomingAccount, outgoingAccount)
+    // if (operation === 'outgoing_payment' || operation === 'incoming_payment') {
+    //   ctx.state.unfulfillable = false
+    // } else {
+    //   ctx.state.unfulfillable = true
+    // }
+    ctx.state.operation = operation
+
     await next()
   }
 }

packages/backend/src/payment-method/ilp/connector/core/middleware/balance.ts

@@ -66,7 +66,17 @@ export function createBalanceMiddleware(): ILPMiddleware {
 
     request.prepare.amount = destinationAmount.toString()
 
+
     if (state.unfulfillable) {
+      logger.info(
+        {
+          amount,
+          destinationAmountOrError,
+          sourceAsset: accounts.incoming.asset,
+          destinationAsset: accounts.outgoing.asset,
+        },
+        'Unfulfillable -> this only should appear during quoting'
+      )
       await next()
       stopTimer()
       return
@@ -86,6 +96,15 @@ export function createBalanceMiddleware(): ILPMiddleware {
       }
       const trxOrError =
         await services.accounting.createTransfer(transferOptions)
+
+      logger.info(
+        {
+          trxOrError,
+          transferOptions,
+          state
+        },
+        'Created transfer'
+      )
       if (isTransferError(trxOrError)) {
         logger.error(
           { transferOptions, transferError: trxOrError },
@@ -122,6 +141,13 @@ export function createBalanceMiddleware(): ILPMiddleware {
       if (trx) {
         if (response.fulfill) {
           await trx.post()
+          logger.info(
+            {
+              trx,
+              response
+            },
+            'Posted transfer'
+          )
         } else {
           await trx.void()
         }