Skip to content

Commit 2d8d484

Browse files
RoasbeefRoasbeef
authored
Merge pull request #980 from lightninglabs/unique-key-fix
tapdb: remove duplicate assets before adding unique index
2 parents 88012a5 + 4a543a3 commit 2d8d484

File tree

3 files changed

+218
-3
lines changed

3 files changed

+218
-3
lines changed

‎tapdb/migrations_test.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"testing"
99

1010
"github.com/btcsuite/btcd/wire"
11+
"github.com/lightninglabs/taproot-assets/asset"
12+
"github.com/lightninglabs/taproot-assets/proof"
1113
"github.com/stretchr/testify/require"
1214
)
1315

@@ -220,3 +222,74 @@ func TestSqliteMigrationBackup(t *testing.T) {
220222
t, wire.TxWitness{{0xbb}}, assets[0].PrevWitnesses[1].TxWitness,
221223
)
222224
}
225+
226+
// TestMigration20 tests that the migration to version 20 works as expected.
227+
// We start at version 19, then insert some test data that simulate duplicated
228+
// assets that might have been created for certain users due to TAP address self
229+
// transfers. The migration with version 20 is then applied, which contains
230+
// SQL queries to de-duplicate the assets, with the goal of then applying a new
231+
// unique constraint on the asset table.
232+
func TestMigration20(t *testing.T) {
233+
ctx := context.Background()
234+
235+
db := NewTestDBWithVersion(t, 19)
236+
237+
// We need to insert some test data that will be affected by the
238+
// migration number 20.
239+
InsertTestdata(t, db.BaseDB, "migrations_test_00020_dummy_data.sql")
240+
241+
// And now that we have test data inserted, we can migrate to the latest
242+
// version.
243+
err := db.ExecuteMigrations(TargetLatest)
244+
require.NoError(t, err)
245+
246+
// The migration should have de-duplicated the assets, so we should now
247+
// only have two valid/distinct assets with two witnesses and one proof
248+
// each. So we're just asserting the expected state _after_ the
249+
// migration has run.
250+
_, assetStore := newAssetStoreFromDB(db.BaseDB)
251+
assets, err := assetStore.FetchAllAssets(ctx, true, false, nil)
252+
require.NoError(t, err)
253+
254+
require.Len(t, assets, 2)
255+
require.Len(t, assets[0].PrevWitnesses, 2)
256+
require.False(t, assets[0].IsSpent)
257+
require.Equal(
258+
t, wire.TxWitness{{0xaa}}, assets[0].PrevWitnesses[0].TxWitness,
259+
)
260+
require.Equal(
261+
t, wire.TxWitness{{0xbb}}, assets[0].PrevWitnesses[1].TxWitness,
262+
)
263+
264+
require.Len(t, assets[1].PrevWitnesses, 2)
265+
require.True(t, assets[1].IsSpent)
266+
require.Equal(
267+
t, wire.TxWitness{{0xcc}}, assets[1].PrevWitnesses[0].TxWitness,
268+
)
269+
require.Equal(
270+
t, wire.TxWitness{{0xdd}}, assets[1].PrevWitnesses[1].TxWitness,
271+
)
272+
273+
asset1Locator := proof.Locator{
274+
ScriptKey: *assets[0].ScriptKey.PubKey,
275+
}
276+
asset1Key := asset.ToSerialized(&asset1Locator.ScriptKey)
277+
asset2Locator := proof.Locator{
278+
ScriptKey: *assets[1].ScriptKey.PubKey,
279+
}
280+
asset2Key := asset.ToSerialized(&asset2Locator.ScriptKey)
281+
282+
p1, err := assetStore.FetchAssetProofs(ctx, asset1Locator)
283+
require.NoError(t, err)
284+
285+
require.Contains(t, p1, asset1Key)
286+
blob1 := p1[asset1Key]
287+
require.Equal(t, []byte{0xaa, 0xaa}, []byte(blob1))
288+
289+
p2, err := assetStore.FetchAssetProofs(ctx, asset2Locator)
290+
require.NoError(t, err)
291+
292+
require.Contains(t, p2, asset2Key)
293+
blob2 := p2[asset2Key]
294+
require.Equal(t, []byte{0xee, 0xee}, []byte(blob2))
295+
}

‎tapdb/sqlc/migrations/000020_asset_unique_key.up.sql‎

Lines changed: 97 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,98 @@
1+
-- Step 1: If the assets were spent, some of the duplicates might not have been
2+
-- updated on that flag. To make sure we can properly group on the spent flag
3+
-- below, we now update all assets that are spent.
4+
UPDATE assets
5+
SET spent = true
6+
WHERE asset_id IN (SELECT a.asset_id
7+
FROM assets a
8+
JOIN managed_utxos mu
9+
ON a.anchor_utxo_id = mu.utxo_id
10+
JOIN chain_txns ct
11+
ON mu.txn_id = ct.txn_id
12+
LEFT JOIN asset_transfer_inputs ati
13+
ON ati.anchor_point = mu.outpoint
14+
WHERE a.spent = false
15+
AND ati.input_id IS NOT NULL);
16+
17+
-- Step 2: Create a temporary table to store the minimum asset_id for each
18+
-- unique combination.
19+
CREATE TABLE tmp_min_assets AS
20+
SELECT MIN(asset_id) AS min_asset_id,
21+
genesis_id,
22+
script_key_id,
23+
amount,
24+
anchor_utxo_id,
25+
spent
26+
FROM assets
27+
GROUP BY genesis_id, script_key_id, amount, anchor_utxo_id, spent;
28+
29+
-- Step 3: Create a mapping table to track old and new asset_ids.
30+
CREATE TABLE tmp_asset_id_mapping AS
31+
SELECT a.asset_id AS old_asset_id,
32+
tmp.min_asset_id AS new_asset_id
33+
FROM assets a
34+
JOIN tmp_min_assets tmp
35+
ON a.genesis_id = tmp.genesis_id
36+
AND a.script_key_id = tmp.script_key_id
37+
AND a.amount = tmp.amount
38+
AND a.anchor_utxo_id = tmp.anchor_utxo_id
39+
AND a.spent = tmp.spent;
40+
41+
-- Step 4: To make the next step possible, we need to disable a unique index on
42+
-- the asset_witnesses table. We'll re-create it later.
43+
DROP INDEX IF EXISTS asset_witnesses_asset_id_witness_index_unique;
44+
45+
-- Step 5: Update the asset_witnesses and asset_proofs tables to reference the
46+
-- new asset_ids.
47+
UPDATE asset_witnesses
48+
SET asset_id = tmp_asset_id_mapping.new_asset_id
49+
FROM tmp_asset_id_mapping
50+
WHERE asset_witnesses.asset_id = tmp_asset_id_mapping.old_asset_id;
51+
52+
-- For the proofs we need skip re-assigning them to the asset that we're going
53+
-- to keep if it already has a proof. This is because the unique index on the
54+
-- asset_proofs table would prevent us from doing so. And we can't disable the
55+
-- unique index, because it is an unnamed/inline index.
56+
UPDATE asset_proofs
57+
SET asset_id = filtered_mapping.new_asset_id
58+
FROM (
59+
SELECT MIN(old_asset_id) AS old_asset_id, new_asset_id
60+
FROM asset_proofs
61+
JOIN tmp_asset_id_mapping
62+
ON asset_proofs.asset_id = tmp_asset_id_mapping.old_asset_id
63+
GROUP BY new_asset_id) AS filtered_mapping
64+
WHERE asset_proofs.asset_id = filtered_mapping.old_asset_id;
65+
66+
-- Step 6: Remove duplicates from the asset_witnesses table.
67+
DELETE
68+
FROM asset_witnesses
69+
WHERE witness_id NOT IN (SELECT min(witness_id)
70+
FROM asset_witnesses
71+
GROUP BY asset_id, witness_index);
72+
73+
-- Step 7: Re-enable the unique index on the asset_witnesses table.
74+
CREATE UNIQUE INDEX asset_witnesses_asset_id_witness_index_unique
75+
ON asset_witnesses (
76+
asset_id, witness_index
77+
);
78+
79+
-- Step 8: Delete any duplicate proofs.
80+
DELETE
81+
FROM asset_proofs
82+
WHERE asset_id NOT IN (SELECT min_asset_id FROM tmp_min_assets);
83+
84+
-- Step 9: Delete the duplicates from the assets table. This will then also
85+
-- delete dangling asset_witnesses.
86+
DELETE
87+
FROM assets
88+
WHERE asset_id NOT IN (SELECT min_asset_id FROM tmp_min_assets);
89+
90+
-- Step 10: Clean up temporary tables.
91+
DROP TABLE IF EXISTS tmp_min_assets;
92+
DROP TABLE IF EXISTS tmp_asset_id_mapping;
93+
94+
-- Step 11: Create the unique index on the assets table.
195
CREATE UNIQUE INDEX assets_genesis_id_script_key_id_anchor_utxo_id_unique
2-
ON assets (
3-
genesis_id, script_key_id, anchor_utxo_id
4-
);
96+
ON assets (
97+
genesis_id, script_key_id, anchor_utxo_id
98+
);

‎tapdb/testdata/migrations_test_00020_dummy_data.sql‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
-- This dummy data inserts duplicate assets, including their witnesses. The
2+
-- migration script with number 20 should clean the duplicates up.
3+
INSERT INTO chain_txns VALUES(1,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f9331',1980,X'02000000000101022cd51ca4d850c5f71ceedf7c50a08ff82d66612b22f631eac95e6b52cbbd2d0000000000ffffffff02e80300000000000022512018ac5a65a0d12e7846c89d24705e2697b1da14627978ba8db24bdbce21fc2aa85cd5f5050000000022512030263d67b4275144b2b00921d220a1311b9a4465fa656ba7d5754b421cb4308402483045022100fa32af97cab8a765dc347c3ff57b14f9810b6dbfc4d02727fb099d1ed875660602204cb66f3bbd92925707158b4aa67338c50a9ffddceb023875eb82b78b3967e007012102eb9cd2a22fd11c40823cb7b0f0fba4156138af69cf73c0644be54f4d46ba480700000000',441,X'4295613d85ccbc455159eb4ddd1e266ca10041d3c75726286b7dfeb3132c9c4f',1);
4+
5+
INSERT INTO genesis_points VALUES(1,X'022cd51ca4d850c5f71ceedf7c50a08ff82d66612b22f631eac95e6b52cbbd2d00000000',1);
6+
7+
INSERT INTO assets_meta VALUES(1,X'2b990b7adb1faf51ccb9b1c73bc5e73926db39cdec8906d4fd3c6c423a3c9821',X'736f6d65206d65746164617461',0);
8+
9+
INSERT INTO genesis_assets VALUES(1,X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04','itestbuxx-collectible',1,0,1,1);
10+
INSERT INTO genesis_assets VALUES(2,X'ffffd0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04','itestbuxx-collectible2',1,0,1,1);
11+
12+
INSERT INTO internal_keys VALUES(1,X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',212,0);
13+
INSERT INTO internal_keys VALUES(2,X'03efbcf2878876bae81ca9a7f6476764d2da38d565b9fb2b691e7bb22fd99f9e5e',212,2);
14+
15+
INSERT INTO managed_utxos VALUES(1,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933100000000',1000,1,X'1dd3e2cf0bbbee32832c4deb57bbae58779fa599be0b8eb1f61e8c624157e2fa',NULL,X'1dd3e2cf0bbbee32832c4deb57bbae58779fa599be0b8eb1f61e8c624157e2fa',1,NULL,NULL,0);
16+
17+
INSERT INTO script_keys VALUES(1,2,X'029c571fffcac1a1a7cd3372bd202ad8562f28e48b90f8a4eb714eca062f576ee6',NULL,true);
18+
INSERT INTO script_keys VALUES(2,2,X'039c571fffcac1a1a7cd3372bd202ad8562f28e48b90f8a4eb714eca062f576ee6',NULL,true);
19+
20+
-- We have some duplicate assets, both having witnesses.
21+
INSERT INTO assets VALUES(1,1,0,1,NULL,0,1,0,0,NULL,NULL,1,false);
22+
INSERT INTO assets VALUES(2,1,0,1,NULL,0,1,0,0,NULL,NULL,1,false);
23+
24+
INSERT INTO asset_witnesses VALUES(1,1,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933100000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101aa',NULL, 0);
25+
INSERT INTO asset_witnesses VALUES(2,1,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933101000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101bb',NULL, 1);
26+
27+
INSERT INTO asset_witnesses VALUES(3,2,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933100000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101cc',NULL, 0);
28+
INSERT INTO asset_witnesses VALUES(4,2,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933101000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101dd',NULL, 1);
29+
30+
INSERT INTO asset_proofs VALUES(1,1,X'aaaa');
31+
32+
INSERT INTO asset_proofs VALUES(2,2,X'cccc');
33+
34+
-- And then a batch of asset duplicates where only one of the duplicates that
35+
-- we're not going to keep has a witness.
36+
INSERT INTO assets VALUES(3,2,0,2,NULL,0,1,0,0,NULL,NULL,1,true);
37+
INSERT INTO assets VALUES(4,2,0,2,NULL,0,1,0,0,NULL,NULL,1,true);
38+
INSERT INTO assets VALUES(5,2,0,2,NULL,0,1,0,0,NULL,NULL,1,true);
39+
40+
INSERT INTO asset_witnesses VALUES(5,5,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933100000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101cc',NULL, 0);
41+
INSERT INTO asset_witnesses VALUES(6,5,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933101000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101dd',NULL, 1);
42+
43+
INSERT INTO asset_witnesses VALUES(7,4,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933100000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101ee',NULL, 0);
44+
INSERT INTO asset_witnesses VALUES(8,4,X'a1594fc379308b2a209f6d0bdb8602e9f87cf71fc232c69032b9a5fed28f933101000000',X'add7d0d7cc37e58a7c0d8ad40b6904050d2baa25a1829f00689c4b27b524dd04',X'02827d74858d152da1fae12010ad8d3c46b595c2d4480512a6575925424617124f',X'0101ff',NULL, 1);
45+
46+
INSERT INTO asset_proofs VALUES(3,4,X'eeee');
47+
48+
INSERT INTO asset_proofs VALUES(4,5,X'eeee');

0 commit comments

Comments
 (0)