Skip to content

Commit c975765

Browse files
tamirmstamirms
authored
services/horizon: Update asset stats processor to ingest new restoration meta (#5611)
1 parent fe9b5ec commit c975765

30 files changed

+258
-387
lines changed

‎.github/workflows/galexie-release.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
# this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
1717
GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5333ec87069efd7bb61f6654a801dc093bf0aad91f43a5ba84806d3efe4a6322
1818
GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
19-
STELLAR_CORE_VERSION: 22.1.0-2194.0241e79f7.focal
19+
STELLAR_CORE_VERSION: 22.1.1-2291.a50f3f919.focal~do~not~use~in~prd
2020
steps:
2121
- name: Set VERSION
2222
run: |

‎.github/workflows/galexie.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
name: Test
1111
runs-on: ubuntu-22.04
1212
env:
13-
CAPTIVE_CORE_DEBIAN_PKG_VERSION: 22.1.0-2194.0241e79f7.focal
13+
CAPTIVE_CORE_DEBIAN_PKG_VERSION: 22.1.1-2291.a50f3f919.focal~do~not~use~in~prd
1414
GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
1515
GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
1616
# this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN

‎.github/workflows/horizon.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ jobs:
3333
HORIZON_INTEGRATION_TESTS_ENABLED: true
3434
HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
3535
HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true
36-
PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.1.0-2194.0241e79f7.focal
37-
PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.1.0-2194.0241e79f7.focal
36+
PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.1.1-2291.a50f3f919.focal~do~not~use~in~prd
37+
PROTOCOL_22_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:22.1.1-2291.a50f3f919.focal-do-not-use-in-prd
3838
PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:22.1.2
3939
PGHOST: localhost
4040
PGPORT: 5432
@@ -124,7 +124,7 @@ jobs:
124124
runs-on: ubuntu-22.04
125125
env:
126126
GO_VERSION: 1.23.4
127-
STELLAR_CORE_VERSION: 22.1.0-2194.0241e79f7.focal
127+
STELLAR_CORE_VERSION: 22.1.1-2291.a50f3f919.jammy~do~not~use~in~prd
128128
CAPTIVE_CORE_STORAGE_PATH: /tmp
129129
steps:
130130
- uses: actions/checkout@v3
@@ -134,7 +134,7 @@ jobs:
134134

135135
- name: Build and test the Verify Range Docker image
136136
run: |
137-
docker build --build-arg="GO_VERSION=$GO_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
137+
docker build --build-arg="GO_VERSION=$GO_VERSION" --build-arg="STELLAR_CORE_VERSION=$STELLAR_CORE_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
138138
# Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
139139
docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
140140

‎ingest/ledgerbackend/captive_core_backend.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
"github.com/stellar/go/xdr"
2121
)
2222

23-
const minProtocolVersionSupported uint = 21
23+
const minProtocolVersionSupported uint = 23
2424

2525
// Ensure CaptiveStellarCore implements LedgerBackend
2626
var _ LedgerBackend = (*CaptiveStellarCore)(nil)

‎ingest/ledgerbackend/captive_core_backend_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ func TestCaptiveNew(t *testing.T) {
160160
HistoryArchiveURLs: historyURLs,
161161
StoragePath: storagePath,
162162
UserAgent: "uatest",
163-
CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil },
163+
CoreProtocolVersionFn: func(string) (uint, error) { return 23, nil },
164164
},
165165
)
166166

@@ -193,11 +193,11 @@ func TestCaptiveNewUnsupportedProtocolVersion(t *testing.T) {
193193
HistoryArchiveURLs: historyURLs,
194194
StoragePath: storagePath,
195195
UserAgent: "uatest",
196-
CoreProtocolVersionFn: func(string) (uint, error) { return 20, nil },
196+
CoreProtocolVersionFn: func(string) (uint, error) { return 22, nil },
197197
},
198198
)
199199

200-
assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 20, but minimum required version is 21. Please upgrade stellar-core to a version that supports protocol version 21 or higher")
200+
assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 22, but minimum required version is 23. Please upgrade stellar-core to a version that supports protocol version 23 or higher")
201201
}
202202

203203
func TestCaptivePrepareRange(t *testing.T) {
@@ -1041,7 +1041,7 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) {
10411041
HistoryArchiveURLs: historyURLs,
10421042
Toml: captiveCoreToml,
10431043
StoragePath: storagePath,
1044-
CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil },
1044+
CoreProtocolVersionFn: func(string) (uint, error) { return 23, nil },
10451045
},
10461046
)
10471047
assert.NoError(t, err)

‎ingest/processors/asset_processor/asset_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
3131
Ext: xdr.TransactionExt{
3232
V: 0,
3333
SorobanData: &xdr.SorobanTransactionData{
34-
Ext: xdr.ExtensionPoint{
35-
V: 0,
36-
},
3734
Resources: xdr.SorobanResources{
3835
Footprint: xdr.LedgerFootprint{
3936
ReadOnly: []xdr.LedgerKey{},

‎ingest/processors/operation_processor/operation_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
7070
Ext: xdr.TransactionExt{
7171
V: 0,
7272
SorobanData: &xdr.SorobanTransactionData{
73-
Ext: xdr.ExtensionPoint{
74-
V: 0,
75-
},
7673
Resources: xdr.SorobanResources{
7774
Footprint: xdr.LedgerFootprint{
7875
ReadOnly: []xdr.LedgerKey{},

‎ingest/processors/trade_processor/trade_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
3333
Ext: xdr.TransactionExt{
3434
V: 0,
3535
SorobanData: &xdr.SorobanTransactionData{
36-
Ext: xdr.ExtensionPoint{
37-
V: 0,
38-
},
3936
Resources: xdr.SorobanResources{
4037
Footprint: xdr.LedgerFootprint{
4138
ReadOnly: []xdr.LedgerKey{},

‎ingest/processors/transaction_processor/transaction_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,6 @@ var genericBumpOperationEnvelope = xdr.TransactionV1Envelope{
6161
Ext: xdr.TransactionExt{
6262
V: 0,
6363
SorobanData: &xdr.SorobanTransactionData{
64-
Ext: xdr.ExtensionPoint{
65-
V: 0,
66-
},
6764
Resources: xdr.SorobanResources{
6865
Footprint: xdr.LedgerFootprint{
6966
ReadOnly: []xdr.LedgerKey{},

‎protocols/horizon/main.go

+5-11
Original file line numberDiff line numberDiff line change
@@ -175,21 +175,15 @@ type AssetStat struct {
175175
} `json:"_links"`
176176

177177
base.Asset
178-
PT string `json:"paging_token"`
179-
ContractID string `json:"contract_id,omitempty"`
180-
NumClaimableBalances int32 `json:"num_claimable_balances"`
181-
NumLiquidityPools int32 `json:"num_liquidity_pools"`
182-
NumContracts int32 `json:"num_contracts"`
183-
// NumArchivedContracts is deprecated and will be removed in the v23 release
184-
// Action needed in release: horizon-v23.0.0: remove field
185-
NumArchivedContracts int32 `json:"num_archived_contracts"`
178+
PT string `json:"paging_token"`
179+
ContractID string `json:"contract_id,omitempty"`
180+
NumClaimableBalances int32 `json:"num_claimable_balances"`
181+
NumLiquidityPools int32 `json:"num_liquidity_pools"`
182+
NumContracts int32 `json:"num_contracts"`
186183
Accounts AssetStatAccounts `json:"accounts"`
187184
ClaimableBalancesAmount string `json:"claimable_balances_amount"`
188185
LiquidityPoolsAmount string `json:"liquidity_pools_amount"`
189186
ContractsAmount string `json:"contracts_amount"`
190-
// ArchivedContractsAmount is deprecated and will be removed in the v23 release
191-
// Action needed in release: horizon-v23.0.0: remove field
192-
ArchivedContractsAmount string `json:"archived_contracts_amount"`
193187
Balances AssetStatBalances `json:"balances"`
194188
Flags AccountFlags `json:"flags"`
195189
}

‎services/horizon/docker/verify-range/Dockerfile

-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ FROM ubuntu:22.04
22

33
ARG GO_VERSION
44
ARG STELLAR_CORE_VERSION
5-
ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*}
65
# to remove tzdata interactive flow
76
ENV DEBIAN_FRONTEND=noninteractive
87

‎services/horizon/docker/verify-range/dependencies

+4-3
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@ apt-get update
77
apt-get install -y curl git libpq-dev libsqlite3-dev libsasl2-dev postgresql-client postgresql postgresql-contrib sudo vim zlib1g-dev wget gnupg2 lsb-release
88
apt-get clean
99

10-
wget -qO - https://apt.stellar.org/SDF.asc | apt-key add -
11-
echo "deb https://apt.stellar.org $(lsb_release -cs) stable" | sudo tee -a /etc/apt/sources.list.d/SDF.list
12-
# echo "deb https://apt.stellar.org $(lsb_release -cs) unstable" | sudo tee -a /etc/apt/sources.list.d/SDF-unstable.list
10+
wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
11+
#echo "deb https://apt.stellar.org $(lsb_release -cs) stable" | sudo tee -a /etc/apt/sources.list.d/SDF.list
12+
echo "deb https://apt.stellar.org $(lsb_release -cs) unstable" | sudo tee -a /etc/apt/sources.list.d/SDF-unstable.list
1313
apt-get update
14+
echo "apt cache-policy ${apt-cache policy stellar-core}"
1415
apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
1516

1617
git clone https://github.com/stellar/go.git stellar-go

‎services/horizon/internal/actions/asset_test.go

-5
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,6 @@ func TestAssetStats(t *testing.T) {
156156
ClaimableBalancesAmount: "0.0000010",
157157
LiquidityPoolsAmount: "0.0000020",
158158
ContractsAmount: "0.0000000",
159-
ArchivedContractsAmount: "0.0000000",
160159
Asset: base.Asset{
161160
Type: "credit_alphanum4",
162161
Code: usdAssetStat.AssetCode,
@@ -199,7 +198,6 @@ func TestAssetStats(t *testing.T) {
199198
ClaimableBalancesAmount: "0.0000000",
200199
LiquidityPoolsAmount: "0.0000000",
201200
ContractsAmount: "0.0000000",
202-
ArchivedContractsAmount: "0.0000000",
203201
Asset: base.Asset{
204202
Type: "credit_alphanum4",
205203
Code: etherAssetStat.AssetCode,
@@ -242,7 +240,6 @@ func TestAssetStats(t *testing.T) {
242240
ClaimableBalancesAmount: "0.0000000",
243241
LiquidityPoolsAmount: "0.0000000",
244242
ContractsAmount: "0.0000000",
245-
ArchivedContractsAmount: "0.0000000",
246243
Asset: base.Asset{
247244
Type: "credit_alphanum4",
248245
Code: otherUSDAssetStat.AssetCode,
@@ -287,7 +284,6 @@ func TestAssetStats(t *testing.T) {
287284
ClaimableBalancesAmount: "0.0000000",
288285
LiquidityPoolsAmount: "0.0000000",
289286
ContractsAmount: "0.0000000",
290-
ArchivedContractsAmount: "0.0000000",
291287
Asset: base.Asset{
292288
Type: "credit_alphanum4",
293289
Code: eurAssetStat.AssetCode,
@@ -461,7 +457,6 @@ func TestAssetStatsIssuerDoesNotExist(t *testing.T) {
461457
ClaimableBalancesAmount: "0.0000000",
462458
LiquidityPoolsAmount: "0.0000000",
463459
ContractsAmount: "0.0000000",
464-
ArchivedContractsAmount: "0.0000000",
465460
Asset: base.Asset{
466461
Type: "credit_alphanum4",
467462
Code: usdAssetStat.AssetCode,

‎services/horizon/internal/db2/history/asset_stats.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -186,11 +186,11 @@ func (q *Q) UpdateContractAssetBalanceExpirations(ctx context.Context, keys []xd
186186
return nil
187187
}
188188

189-
// GetContractAssetBalancesExpiringAt returns all contract asset balances which are active
189+
// DeleteContractAssetBalancesExpiringAt deletes and returns all contract asset balances which are active
190190
// at `ledger` and expired at `ledger+1`
191-
func (q *Q) GetContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) {
192-
sql := sq.Select("contract_asset_balances.*").From("contract_asset_balances").
193-
Where(map[string]interface{}{"expiration_ledger": ledger})
191+
func (q *Q) DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error) {
192+
sql := sq.Delete("contract_asset_balances").
193+
Where(map[string]interface{}{"expiration_ledger": ledger}).Suffix("RETURNING *")
194194
var balances []ContractAssetBalance
195195
err := q.Select(ctx, &balances, sql)
196196
return balances, err

‎services/horizon/internal/db2/history/asset_stats_test.go

+20-35
Original file line numberDiff line numberDiff line change
@@ -125,28 +125,22 @@ func TestAssetContractStats(t *testing.T) {
125125
c1 := ContractAssetStatRow{
126126
ContractID: []byte{1},
127127
Stat: ContractStat{
128-
ActiveBalance: "100",
129-
ActiveHolders: 2,
130-
ArchivedBalance: "0",
131-
ArchivedHolders: 0,
128+
ActiveBalance: "100",
129+
ActiveHolders: 2,
132130
},
133131
}
134132
c2 := ContractAssetStatRow{
135133
ContractID: []byte{2},
136134
Stat: ContractStat{
137-
ActiveBalance: "40",
138-
ActiveHolders: 1,
139-
ArchivedBalance: "0",
140-
ArchivedHolders: 0,
135+
ActiveBalance: "40",
136+
ActiveHolders: 1,
141137
},
142138
}
143139
c3 := ContractAssetStatRow{
144140
ContractID: []byte{3},
145141
Stat: ContractStat{
146-
ActiveBalance: "900",
147-
ActiveHolders: 12,
148-
ArchivedBalance: "23",
149-
ArchivedHolders: 3,
142+
ActiveBalance: "900",
143+
ActiveHolders: 12,
150144
},
151145
}
152146

@@ -161,7 +155,6 @@ func TestAssetContractStats(t *testing.T) {
161155

162156
c2.Stat.ActiveHolders = 3
163157
c2.Stat.ActiveBalance = "20"
164-
c3.Stat.ArchivedBalance = "900"
165158
c2.Stat.ActiveHolders = 5
166159
numRows, err := q.UpdateContractAssetStat(tt.Ctx, c2)
167160
tt.Assert.NoError(err)
@@ -568,10 +561,8 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) {
568561

569562
q := &Q{tt.HorizonSession()}
570563
zero := ContractStat{
571-
ActiveBalance: "0",
572-
ActiveHolders: 0,
573-
ArchivedBalance: "0",
574-
ArchivedHolders: 0,
564+
ActiveBalance: "0",
565+
ActiveHolders: 0,
575566
}
576567
usdAssetStat := AssetAndContractStat{
577568
ExpAssetStat: ExpAssetStat{
@@ -652,10 +643,8 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) {
652643
},
653644
},
654645
Contracts: ContractStat{
655-
ActiveBalance: "120",
656-
ActiveHolders: 3,
657-
ArchivedBalance: "90",
658-
ArchivedHolders: 1,
646+
ActiveBalance: "120",
647+
ActiveHolders: 3,
659648
},
660649
}
661650
eurAssetStat.SetContractID([32]byte{})
@@ -684,10 +673,8 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) {
684673
numChanged, err := q.InsertContractAssetStat(tt.Ctx, ContractAssetStatRow{
685674
ContractID: []byte{1},
686675
Stat: ContractStat{
687-
ActiveBalance: "400",
688-
ActiveHolders: 30,
689-
ArchivedBalance: "0",
690-
ArchivedHolders: 0,
676+
ActiveBalance: "400",
677+
ActiveHolders: 30,
691678
},
692679
})
693680
tt.Assert.NoError(err)
@@ -1141,14 +1128,18 @@ func TestUpdateContractAssetBalanceExpirations(t *testing.T) {
11411128
q.InsertContractAssetBalances(context.Background(), []ContractAssetBalance{balance, otherBalance}),
11421129
)
11431130

1144-
balances, err := q.GetContractAssetBalancesExpiringAt(context.Background(), 10)
1131+
balances, err := q.DeleteContractAssetBalancesExpiringAt(context.Background(), 10)
11451132
tt.Assert.NoError(err)
11461133
assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance})
11471134

1148-
balances, err = q.GetContractAssetBalancesExpiringAt(context.Background(), 11)
1135+
balances, err = q.DeleteContractAssetBalancesExpiringAt(context.Background(), 11)
11491136
tt.Assert.NoError(err)
11501137
assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{otherBalance})
11511138

1139+
balances, err = q.GetContractAssetBalances(context.Background(), []xdr.Hash{keyHash, otherKeyHash})
1140+
tt.Assert.NoError(err)
1141+
tt.Assert.Empty(balances)
1142+
11521143
nonExistantKeyHash := xdr.Hash{4}
11531144

11541145
tt.Assert.NoError(
@@ -1161,17 +1152,11 @@ func TestUpdateContractAssetBalanceExpirations(t *testing.T) {
11611152

11621153
balances, err = q.GetContractAssetBalances(context.Background(), []xdr.Hash{keyHash, otherKeyHash})
11631154
tt.Assert.NoError(err)
1164-
balance.ExpirationLedger = 200
1165-
otherBalance.ExpirationLedger = 200
1166-
assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance, otherBalance})
1155+
tt.Assert.Empty(balances)
11671156

1168-
balances, err = q.GetContractAssetBalancesExpiringAt(context.Background(), 10)
1157+
balances, err = q.DeleteContractAssetBalancesExpiringAt(context.Background(), 10)
11691158
tt.Assert.NoError(err)
11701159
assert.Empty(t, balances)
11711160

1172-
balances, err = q.GetContractAssetBalancesExpiringAt(context.Background(), 200)
1173-
tt.Assert.NoError(err)
1174-
assertContractAssetBalancesEqual(t, balances, []ContractAssetBalance{balance, otherBalance})
1175-
11761161
tt.Assert.NoError(q.Rollback())
11771162
}

‎services/horizon/internal/db2/history/main.go

+3-9
Original file line numberDiff line numberDiff line change
@@ -376,10 +376,8 @@ type Asset struct {
376376
}
377377

378378
type ContractStat struct {
379-
ActiveBalance string `json:"balance"`
380-
ActiveHolders int32 `json:"holders"`
381-
ArchivedBalance string `json:"archived_balance"`
382-
ArchivedHolders int32 `json:"archived_holders"`
379+
ActiveBalance string `json:"balance"`
380+
ActiveHolders int32 `json:"holders"`
383381
}
384382

385383
func (c ContractStat) Value() (driver.Value, error) {
@@ -393,7 +391,6 @@ func (c ContractStat) Value() (driver.Value, error) {
393391
func (c *ContractStat) Scan(src interface{}) error {
394392
if src == nil {
395393
c.ActiveBalance = "0"
396-
c.ArchivedBalance = "0"
397394
return nil
398395
}
399396

@@ -411,9 +408,6 @@ func (c *ContractStat) Scan(src interface{}) error {
411408
if c.ActiveBalance == "" {
412409
c.ActiveBalance = "0"
413410
}
414-
if c.ArchivedBalance == "" {
415-
c.ArchivedBalance = "0"
416-
}
417411

418412
return nil
419413
}
@@ -581,7 +575,7 @@ type QAssetStats interface {
581575
UpdateContractAssetBalanceAmounts(ctx context.Context, keys []xdr.Hash, amounts []string) error
582576
UpdateContractAssetBalanceExpirations(ctx context.Context, keys []xdr.Hash, expirationLedgers []uint32) error
583577
GetContractAssetBalances(ctx context.Context, keys []xdr.Hash) ([]ContractAssetBalance, error)
584-
GetContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error)
578+
DeleteContractAssetBalancesExpiringAt(ctx context.Context, ledger uint32) ([]ContractAssetBalance, error)
585579
InsertAssetStats(ctx context.Context, stats []ExpAssetStat) error
586580
InsertContractAssetStats(ctx context.Context, rows []ContractAssetStatRow) error
587581
InsertAssetStat(ctx context.Context, stat ExpAssetStat) (int64, error)

0 commit comments

Comments
 (0)