services/horizon: Update asset stats processor to ingest new restoration meta

.github/workflows/galexie-release.yml

@@ -16,7 +16,7 @@ jobs:
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
       GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5333ec87069efd7bb61f6654a801dc093bf0aad91f43a5ba84806d3efe4a6322
       GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
-      STELLAR_CORE_VERSION: 22.1.0-2194.0241e79f7.focal
+      STELLAR_CORE_VERSION: 22.1.1-2291.a50f3f919.focal~do~not~use~in~prd
     steps:
       - name: Set VERSION
         run: |

.github/workflows/galexie.yml

@@ -10,7 +10,7 @@ jobs:
     name: Test
     runs-on: ubuntu-22.04
     env:
-      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 22.1.0-2194.0241e79f7.focal
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 22.1.1-2291.a50f3f919.focal~do~not~use~in~prd
       GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
       GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN

.github/workflows/horizon.yml

@@ -33,8 +33,8 @@ jobs:
       HORIZON_INTEGRATION_TESTS_ENABLED: true
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
       HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true
-      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.1.0-2194.0241e79f7.focal
-      PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.1.0-2194.0241e79f7.focal
+      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.1.1-2291.a50f3f919.focal~do~not~use~in~prd
+      PROTOCOL_22_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:22.1.1-2291.a50f3f919.focal-do-not-use-in-prd
       PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:22.1.2
       PGHOST: localhost
       PGPORT: 5432
@@ -124,7 +124,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       GO_VERSION: 1.23.4
-      STELLAR_CORE_VERSION: 22.1.0-2194.0241e79f7.focal
+      STELLAR_CORE_VERSION: 22.1.1-2291.a50f3f919.jammy~do~not~use~in~prd
       CAPTIVE_CORE_STORAGE_PATH: /tmp
     steps:
       - uses: actions/checkout@v3
@@ -134,7 +134,7 @@ jobs:
 
       - name: Build and test the Verify Range Docker image
         run: |
-          docker build --build-arg="GO_VERSION=$GO_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
+          docker build --build-arg="GO_VERSION=$GO_VERSION" --build-arg="STELLAR_CORE_VERSION=$STELLAR_CORE_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
           # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
           docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
 

ingest/ledgerbackend/captive_core_backend.go

@@ -20,7 +20,7 @@ import (
 	"github.com/stellar/go/xdr"
 )
 
-const minProtocolVersionSupported uint = 21
+const minProtocolVersionSupported uint = 23
 
 // Ensure CaptiveStellarCore implements LedgerBackend
 var _ LedgerBackend = (*CaptiveStellarCore)(nil)

ingest/ledgerbackend/captive_core_backend_test.go

@@ -160,7 +160,7 @@ func TestCaptiveNew(t *testing.T) {
 			HistoryArchiveURLs:    historyURLs,
 			StoragePath:           storagePath,
 			UserAgent:             "uatest",
-			CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil },
+			CoreProtocolVersionFn: func(string) (uint, error) { return 23, nil },
 		},
 	)
 
@@ -193,11 +193,11 @@ func TestCaptiveNewUnsupportedProtocolVersion(t *testing.T) {
 			HistoryArchiveURLs:    historyURLs,
 			StoragePath:           storagePath,
 			UserAgent:             "uatest",
-			CoreProtocolVersionFn: func(string) (uint, error) { return 20, nil },
+			CoreProtocolVersionFn: func(string) (uint, error) { return 22, nil },
 		},
 	)
 
-	assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 20, but minimum required version is 21. Please upgrade stellar-core to a version that supports protocol version 21 or higher")
+	assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 22, but minimum required version is 23. Please upgrade stellar-core to a version that supports protocol version 23 or higher")
 }
 
 func TestCaptivePrepareRange(t *testing.T) {
@@ -1041,7 +1041,7 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) {
 			HistoryArchiveURLs:    historyURLs,
 			Toml:                  captiveCoreToml,
 			StoragePath:           storagePath,
-			CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil },
+			CoreProtocolVersionFn: func(string) (uint, error) { return 23, nil },
 		},
 	)
 	assert.NoError(t, err)

ingest/processors/asset_processor/asset_test.go

@@ -31,9 +31,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
 		Ext: xdr.TransactionExt{
 			V: 0,
 			SorobanData: &xdr.SorobanTransactionData{
-				Ext: xdr.ExtensionPoint{
-					V: 0,
-				},
 				Resources: xdr.SorobanResources{
 					Footprint: xdr.LedgerFootprint{
 						ReadOnly:  []xdr.LedgerKey{},

ingest/processors/operation_processor/operation_test.go

@@ -70,9 +70,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
 		Ext: xdr.TransactionExt{
 			V: 0,
 			SorobanData: &xdr.SorobanTransactionData{
-				Ext: xdr.ExtensionPoint{
-					V: 0,
-				},
 				Resources: xdr.SorobanResources{
 					Footprint: xdr.LedgerFootprint{
 						ReadOnly:  []xdr.LedgerKey{},

ingest/processors/trade_processor/trade_test.go

@@ -33,9 +33,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
 		Ext: xdr.TransactionExt{
 			V: 0,
 			SorobanData: &xdr.SorobanTransactionData{
-				Ext: xdr.ExtensionPoint{
-					V: 0,
-				},
 				Resources: xdr.SorobanResources{
 					Footprint: xdr.LedgerFootprint{
 						ReadOnly:  []xdr.LedgerKey{},

ingest/processors/transaction_processor/transaction_test.go

@@ -61,9 +61,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
 		Ext: xdr.TransactionExt{
 			V: 0,
 			SorobanData: &xdr.SorobanTransactionData{
-				Ext: xdr.ExtensionPoint{
-					V: 0,
-				},
 				Resources: xdr.SorobanResources{
 					Footprint: xdr.LedgerFootprint{
 						ReadOnly:  []xdr.LedgerKey{},

protocols/horizon/main.go

@@ -175,21 +175,15 @@ type AssetStat struct {
 	} `json:"_links"`
 
 	base.Asset
-	PT                   string `json:"paging_token"`
-	ContractID           string `json:"contract_id,omitempty"`
-	NumClaimableBalances int32  `json:"num_claimable_balances"`
-	NumLiquidityPools    int32  `json:"num_liquidity_pools"`
-	NumContracts         int32  `json:"num_contracts"`
-	// NumArchivedContracts is deprecated and will be removed in the v23 release
-	// Action needed in release: horizon-v23.0.0: remove field
-	NumArchivedContracts    int32             `json:"num_archived_contracts"`
+	PT                      string            `json:"paging_token"`
+	ContractID              string            `json:"contract_id,omitempty"`
+	NumClaimableBalances    int32             `json:"num_claimable_balances"`
+	NumLiquidityPools       int32             `json:"num_liquidity_pools"`
+	NumContracts            int32             `json:"num_contracts"`
 	Accounts                AssetStatAccounts `json:"accounts"`
 	ClaimableBalancesAmount string            `json:"claimable_balances_amount"`
 	LiquidityPoolsAmount    string            `json:"liquidity_pools_amount"`
 	ContractsAmount         string            `json:"contracts_amount"`
-	// ArchivedContractsAmount is deprecated and will be removed in the v23 release
-	// Action needed in release: horizon-v23.0.0: remove field
-	ArchivedContractsAmount string            `json:"archived_contracts_amount"`
 	Balances                AssetStatBalances `json:"balances"`
 	Flags                   AccountFlags      `json:"flags"`
 }

services/horizon/docker/verify-range/Dockerfile

@@ -2,7 +2,6 @@ FROM ubuntu:22.04
 
 ARG GO_VERSION
 ARG STELLAR_CORE_VERSION 
-ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*}
 # to remove tzdata interactive flow
 ENV DEBIAN_FRONTEND=noninteractive
 

services/horizon/docker/verify-range/dependencies

@@ -7,10 +7,11 @@ apt-get update
 apt-get install -y curl git libpq-dev libsqlite3-dev libsasl2-dev postgresql-client postgresql postgresql-contrib sudo vim zlib1g-dev wget gnupg2 lsb-release
 apt-get clean
 
-wget -qO - https://apt.stellar.org/SDF.asc | apt-key add -
-echo "deb https://apt.stellar.org $(lsb_release -cs) stable" | sudo tee -a /etc/apt/sources.list.d/SDF.list
-# echo "deb https://apt.stellar.org $(lsb_release -cs) unstable" | sudo tee -a /etc/apt/sources.list.d/SDF-unstable.list
+wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+#echo "deb https://apt.stellar.org $(lsb_release -cs) stable" | sudo tee -a /etc/apt/sources.list.d/SDF.list
+echo "deb https://apt.stellar.org $(lsb_release -cs) unstable" | sudo tee -a /etc/apt/sources.list.d/SDF-unstable.list
 apt-get update
+echo "apt cache-policy ${apt-cache policy stellar-core}"
 apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 
 git clone https://github.com/stellar/go.git stellar-go

services/horizon/internal/actions/asset_test.go

@@ -156,7 +156,6 @@ func TestAssetStats(t *testing.T) {
 		ClaimableBalancesAmount: "0.0000010",
 		LiquidityPoolsAmount:    "0.0000020",
 		ContractsAmount:         "0.0000000",
-		ArchivedContractsAmount: "0.0000000",
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   usdAssetStat.AssetCode,
@@ -199,7 +198,6 @@ func TestAssetStats(t *testing.T) {
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
 		ContractsAmount:         "0.0000000",
-		ArchivedContractsAmount: "0.0000000",
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   etherAssetStat.AssetCode,
@@ -242,7 +240,6 @@ func TestAssetStats(t *testing.T) {
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
 		ContractsAmount:         "0.0000000",
-		ArchivedContractsAmount: "0.0000000",
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   otherUSDAssetStat.AssetCode,
@@ -287,7 +284,6 @@ func TestAssetStats(t *testing.T) {
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
 		ContractsAmount:         "0.0000000",
-		ArchivedContractsAmount: "0.0000000",
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   eurAssetStat.AssetCode,
@@ -461,7 +457,6 @@ func TestAssetStatsIssuerDoesNotExist(t *testing.T) {
 		ClaimableBalancesAmount: "0.0000000",
 		LiquidityPoolsAmount:    "0.0000000",
 		ContractsAmount:         "0.0000000",
-		ArchivedContractsAmount: "0.0000000",
 		Asset: base.Asset{
 			Type:   "credit_alphanum4",
 			Code:   usdAssetStat.AssetCode,

services/horizon/internal/db2/history/asset_stats.go

@@ -186,11 +186,11 @@ func (q *Q) UpdateContractAssetBalanceExpirations(ctx context.Context, keys []xd
 	return nil
 }
 
-// GetContractAssetBalancesExpiringAt returns all contract asset balances which are active
+// DeleteContractAssetBalancesExpiringAt deletes and returns all contract asset balances which are active
 // at `ledger` and expired at `ledger+1`
-func (q *Q) GetContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) {
-	sql := sq.Select("contract_asset_balances.*").From("contract_asset_balances").
-		Where(map[string]interface{}{"expiration_ledger": ledger})
+func (q *Q) DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) {
+	sql := sq.Delete("contract_asset_balances").
+		Where(map[string]interface{}{"expiration_ledger": ledger}).Suffix("RETURNING *")
 	var balances []ContractAssetBalance
 	err := q.Select(ctx, &balances, sql)
 	return balances, err

services/horizon/internal/db2/history/asset_stats_test.go

@@ -125,28 +125,22 @@ func TestAssetContractStats(t *testing.T) {
 	c1 := ContractAssetStatRow{
 		ContractID: []byte{1},
 		Stat: ContractStat{
-			ActiveBalance:   "100",
-			ActiveHolders:   2,
-			ArchivedBalance: "0",
-			ArchivedHolders: 0,
+			ActiveBalance: "100",
+			ActiveHolders: 2,
 		},
 	}
 	c2 := ContractAssetStatRow{
 		ContractID: []byte{2},
 		Stat: ContractStat{
-			ActiveBalance:   "40",
-			ActiveHolders:   1,
-			ArchivedBalance: "0",
-			ArchivedHolders: 0,
+			ActiveBalance: "40",
+			ActiveHolders: 1,
 		},
 	}
 	c3 := ContractAssetStatRow{
 		ContractID: []byte{3},
 		Stat: ContractStat{
-			ActiveBalance:   "900",
-			ActiveHolders:   12,
-			ArchivedBalance: "23",
-			ArchivedHolders: 3,
+			ActiveBalance: "900",
+			ActiveHolders: 12,
 		},
 	}
 
@@ -161,7 +155,6 @@ func TestAssetContractStats(t *testing.T) {
 
 	c2.Stat.ActiveHolders = 3
 	c2.Stat.ActiveBalance = "20"
-	c3.Stat.ArchivedBalance = "900"
 	c2.Stat.ActiveHolders = 5
 	numRows, err := q.UpdateContractAssetStat(tt.Ctx, c2)
 	tt.Assert.NoError(err)
@@ -568,10 +561,8 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) {
 
 	q := &Q{tt.HorizonSession()}
 	zero := ContractStat{
-		ActiveBalance:   "0",
-		ActiveHolders:   0,
-		ArchivedBalance: "0",
-		ArchivedHolders: 0,
+		ActiveBalance: "0",
+		ActiveHolders: 0,
 	}
 	usdAssetStat := AssetAndContractStat{
 		ExpAssetStat: ExpAssetStat{
@@ -652,10 +643,8 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) {
 			},
 		},
 		Contracts: ContractStat{
-			ActiveBalance:   "120",
-			ActiveHolders:   3,
-			ArchivedBalance: "90",
-			ArchivedHolders: 1,
+			ActiveBalance: "120",
+			ActiveHolders: 3,
 		},
 	}
 	eurAssetStat.SetContractID([32]byte{})
@@ -684,10 +673,8 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) {
 	numChanged, err := q.InsertContractAssetStat(tt.Ctx, ContractAssetStatRow{
 		ContractID: []byte{1},
 		Stat: ContractStat{
-			ActiveBalance:   "400",
-			ActiveHolders:   30,
-			ArchivedBalance: "0",
-			ArchivedHolders: 0,
+			ActiveBalance: "400",
+			ActiveHolders: 30,
 		},
 	})
 	tt.Assert.NoError(err)
@@ -1141,14 +1128,18 @@ func TestUpdateContractAssetBalanceExpirations(t *testing.T) {
 		q.InsertContractAssetBalances(context.Background(), []ContractAssetBalance{balance, otherBalance}),
 	)
 
-	balances, err := q.GetContractAssetBalancesExpiringAt(context.Background(), 10)
+	balances, err := q.DeleteContractAssetBalancesExpiringAt(context.Background(), 10)
 	tt.Assert.NoError(err)
 	assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance})
 
-	balances, err = q.GetContractAssetBalancesExpiringAt(context.Background(), 11)
+	balances, err = q.DeleteContractAssetBalancesExpiringAt(context.Background(), 11)
 	tt.Assert.NoError(err)
 	assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{otherBalance})
 
+	balances, err = q.GetContractAssetBalances(context.Background(), []xdr.Hash{keyHash, otherKeyHash})
+	tt.Assert.NoError(err)
+	tt.Assert.Empty(balances)
+
 	nonExistantKeyHash := xdr.Hash{4}
 
 	tt.Assert.NoError(
@@ -1161,17 +1152,11 @@ func TestUpdateContractAssetBalanceExpirations(t *testing.T) {
 
 	balances, err = q.GetContractAssetBalances(context.Background(), []xdr.Hash{keyHash, otherKeyHash})
 	tt.Assert.NoError(err)
-	balance.ExpirationLedger = 200
-	otherBalance.ExpirationLedger = 200
-	assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance, otherBalance})
+	tt.Assert.Empty(balances)
 
-	balances, err = q.GetContractAssetBalancesExpiringAt(context.Background(), 10)
+	balances, err = q.DeleteContractAssetBalancesExpiringAt(context.Background(), 10)
 	tt.Assert.NoError(err)
 	assert.Empty(t, balances)
 
-	balances, err = q.GetContractAssetBalancesExpiringAt(context.Background(), 200)
-	tt.Assert.NoError(err)
-	assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance, otherBalance})
-
 	tt.Assert.NoError(q.Rollback())
 }

services/horizon/internal/db2/history/main.go

@@ -376,10 +376,8 @@ type Asset struct {
 }
 
 type ContractStat struct {
-	ActiveBalance   string `json:"balance"`
-	ActiveHolders   int32  `json:"holders"`
-	ArchivedBalance string `json:"archived_balance"`
-	ArchivedHolders int32  `json:"archived_holders"`
+	ActiveBalance string `json:"balance"`
+	ActiveHolders int32  `json:"holders"`
 }
 
 func (c ContractStat) Value() (driver.Value, error) {
@@ -393,7 +391,6 @@ func (c ContractStat) Value() (driver.Value, error) {
 func (c *ContractStat) Scan(src interface{}) error {
 	if src == nil {
 		c.ActiveBalance = "0"
-		c.ArchivedBalance = "0"
 		return nil
 	}
 
@@ -411,9 +408,6 @@ func (c *ContractStat) Scan(src interface{}) error {
 	if c.ActiveBalance == "" {
 		c.ActiveBalance = "0"
 	}
-	if c.ArchivedBalance == "" {
-		c.ArchivedBalance = "0"
-	}
 
 	return nil
 }
@@ -581,7 +575,7 @@ type QAssetStats interface {
 	UpdateContractAssetBalanceAmounts(ctx context.Context, keys []xdr.Hash, amounts []string) error
 	UpdateContractAssetBalanceExpirations(ctx context.Context, keys []xdr.Hash, expirationLedgers []uint32) error
 	GetContractAssetBalances(ctx context.Context, keys []xdr.Hash) ([]ContractAssetBalance, error)
-	GetContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error)
+	DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error)
 	InsertAssetStats(ctx context.Context, stats []ExpAssetStat) error
 	InsertContractAssetStats(ctx context.Context, rows []ContractAssetStatRow) error
 	InsertAssetStat(ctx context.Context, stat ExpAssetStat) (int64, error)