From efec46f6c6b03acfafea3ecaf106438b3277150f Mon Sep 17 00:00:00 2001 From: "Andrew Camilleri (Kukks)" Date: Mon, 25 Aug 2025 22:41:46 +0200 Subject: [PATCH 1/3] Support all tracked sources for scanning utxo set and offer Get Addresses API --- NBXplorer.Client/ExplorerClient.cs | 27 +++- NBXplorer/Backend/Repository.cs | 10 ++ .../Controllers/CommonRoutesController.cs | 39 +++++- .../DerivationSchemesController.cs | 38 ----- NBXplorer/Controllers/GroupsController.cs | 7 + NBXplorer/ScanUTXOSetService.cs | 130 +++++++++++------- 6 files changed, 155 insertions(+), 96 deletions(-) diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 63281fa40..4e0ba0581 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -182,10 +182,17 @@ public RawStr(string str) public override string ToString() => str; }; internal static RawStr Raw(string str) => new RawStr(str); - public async Task ScanUTXOSetAsync(DerivationStrategyBase extKey, int? batchSize = null, int? gapLimit = null, int? fromIndex = null, CancellationToken cancellation = default) + + public async Task ScanUTXOSetAsync(DerivationStrategyBase extKey, int? batchSize = null, int? gapLimit = null, + int? fromIndex = null, CancellationToken cancellation = default) { - if (extKey == null) - throw new ArgumentNullException(nameof(extKey)); + await ScanUTXOSetAsync(new DerivationSchemeTrackedSource(extKey), batchSize, gapLimit, fromIndex, + cancellation); + } + public async Task ScanUTXOSetAsync(TrackedSource trackedSource, int? batchSize = null, int? gapLimit = null, int? fromIndex = null, CancellationToken cancellation = default) + { + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); List args = new List(); if (batchSize != null) args.Add($"batchsize={batchSize.Value}"); @@ -196,16 +203,24 @@ public async Task ScanUTXOSetAsync(DerivationStrategyBase extKey, int? batchSize var argsString = string.Join("&", args.ToArray()); if (argsString != string.Empty) argsString = $"?{argsString}"; - await SendAsync(HttpMethod.Post, null, $"v1/cryptos/{CryptoCode}/derivations/{extKey}/utxos/scan{Raw(argsString)}", cancellation).ConfigureAwait(false); + + + await SendAsync(HttpMethod.Post, null, $"{GetBasePath(trackedSource)}/utxos/scan{Raw(argsString)}", cancellation).ConfigureAwait(false); } public void ScanUTXOSet(DerivationStrategyBase extKey, int? batchSize = null, int? gapLimit = null, int? fromIndex = null, CancellationToken cancellation = default) { ScanUTXOSetAsync(extKey, batchSize, gapLimit, fromIndex, cancellation).GetAwaiter().GetResult(); } - public async Task GetScanUTXOSetInformationAsync(DerivationStrategyBase extKey, CancellationToken cancellation = default) + public async Task GetScanUTXOSetInformationAsync(DerivationStrategyBase extKey, + CancellationToken cancellation = default) + { + return await GetScanUTXOSetInformationAsync(new DerivationSchemeTrackedSource(extKey), cancellation); + } + + public async Task GetScanUTXOSetInformationAsync(TrackedSource trackedSource, CancellationToken cancellation = default) { - return await SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{extKey}/utxos/scan", cancellation).ConfigureAwait(false); + return await SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/utxos/scan", cancellation).ConfigureAwait(false); } public ScanUTXOInformation GetScanUTXOSetInformation(DerivationStrategyBase extKey, CancellationToken cancellation = default) diff --git a/NBXplorer/Backend/Repository.cs b/NBXplorer/Backend/Repository.cs index 6e2b80077..e734a7703 100644 --- a/NBXplorer/Backend/Repository.cs +++ b/NBXplorer/Backend/Repository.cs @@ -1320,5 +1320,15 @@ public void RemoveFromCache(IEnumerable txIds) internal static readonly string WalletInsertQuery = "INSERT INTO wallets (wallet_id, metadata) VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING;"; internal static readonly string WalletCheckQuery = "SELECT COUNT(*) FROM wallets WHERE wallet_id=@wid"; + + public async Task GetAddresses(TrackedSource trackedSource, NBXplorerNetwork network) + { + var walletKey = GetWalletKey(trackedSource, network); + await using var conn = await ConnectionFactory.CreateConnection(); + return (await conn.QueryAsync("SELECT s.addr FROM wallets_scripts JOIN scripts s USING (code, script) WHERE code=@code AND wallet_id=@wid", new + { + code = network.CryptoCode, walletKey.wid + })).ToArray(); + } } } diff --git a/NBXplorer/Controllers/CommonRoutesController.cs b/NBXplorer/Controllers/CommonRoutesController.cs index 63ff16343..70d05a2b0 100644 --- a/NBXplorer/Controllers/CommonRoutesController.cs +++ b/NBXplorer/Controllers/CommonRoutesController.cs @@ -22,16 +22,53 @@ namespace NBXplorer.Controllers [Authorize] public class CommonRoutesController : Controller { + public ScanUTXOSetService ScanUTXOSetService { get; } public GroupsController GroupsController{ get; } public AddressPoolService AddressPoolService{ get; } public DbConnectionFactory ConnectionFactory { get; } - public CommonRoutesController(DbConnectionFactory connectionFactory, AddressPoolService addressPoolService, GroupsController groupsController) + public CommonRoutesController( + DbConnectionFactory connectionFactory, + AddressPoolService addressPoolService, + GroupsController groupsController, + ScanUTXOSetServiceAccessor scanUTXOSetService) { + ScanUTXOSetService = scanUTXOSetService.Instance; GroupsController = groupsController; AddressPoolService = addressPoolService; ConnectionFactory = connectionFactory; } + [HttpPost("utxos/scan")] + [TrackedSourceContext.TrackedSourceContextRequirement(requireRPC: true)] + public IActionResult ScanUTXOSet(TrackedSourceContext trackedSourceContext, int? batchSize = null, int? gapLimit = null, int? from = null) + { + var network = trackedSourceContext.Network; + var rpc = trackedSourceContext.RpcClient; + if (!rpc.Capabilities.SupportScanUTXOSet) + throw new NBXplorerError(405, "scanutxoset-not-suported", "ScanUTXOSet is not supported for this currency").AsException(); + + ScanUTXOSetOptions options = new ScanUTXOSetOptions(); + if (batchSize != null) + options.BatchSize = batchSize.Value; + if (gapLimit != null) + options.GapLimit = gapLimit.Value; + if (from != null) + options.From = from.Value; + if (!ScanUTXOSetService.EnqueueScan(network, trackedSourceContext.TrackedSource, options)) + throw new NBXplorerError(409, "scanutxoset-in-progress", "ScanUTXOSet has already been called for this derivationScheme").AsException(); + return Ok(); + } + + [HttpGet($"utxos/scan")] + [TrackedSourceContext.TrackedSourceContextRequirement()] + public IActionResult GetScanUTXOSetInformation(TrackedSourceContext trackedSourceContext) + { + var network = trackedSourceContext.Network; + var info = ScanUTXOSetService.GetInformation(network, trackedSourceContext.TrackedSource); + if (info == null) + throw new NBXplorerError(404, "scanutxoset-info-not-found", "ScanUTXOSet has not been called with this derivationScheme of the result has expired").AsException(); + return Json(info, network.Serializer.Settings); + } [HttpGet("")] public async Task IsTracked(TrackedSourceContext trackedSourceContext) diff --git a/NBXplorer/Controllers/DerivationSchemesController.cs b/NBXplorer/Controllers/DerivationSchemesController.cs index d785d49b8..5e2c9cdb4 100644 --- a/NBXplorer/Controllers/DerivationSchemesController.cs +++ b/NBXplorer/Controllers/DerivationSchemesController.cs @@ -19,7 +19,6 @@ namespace NBXplorer.Controllers [Route($"v1/{CommonRoutes.BaseDerivationEndpoint}")] public class DerivationSchemesController : Controller { - public ScanUTXOSetService ScanUTXOSetService { get; } public MainController MainController { get; } public RepositoryProvider RepositoryProvider { get; } public KeyPathTemplates KeyPathTemplates { get; } @@ -27,13 +26,11 @@ public class DerivationSchemesController : Controller public AddressPoolService AddressPoolService { get; } public DerivationSchemesController( MainController mainController, - ScanUTXOSetServiceAccessor scanUTXOSetService, RepositoryProvider repositoryProvider, KeyPathTemplates keyPathTemplates, Indexers indexers, AddressPoolService addressPoolService) { - ScanUTXOSetService = scanUTXOSetService.Instance; MainController = mainController; RepositoryProvider = repositoryProvider; KeyPathTemplates = keyPathTemplates; @@ -104,41 +101,6 @@ public async Task Wipe(TrackedSourceContext trackedSourceContext) return Ok(); } - [HttpPost("utxos/scan")] - [HttpPost($"~/v1/{CommonRoutes.DerivationEndpoint}/utxos/scan")] - [TrackedSourceContext.TrackedSourceContextRequirement(requireRPC: true, allowedTrackedSourceTypes: typeof(DerivationSchemeTrackedSource))] - public IActionResult ScanUTXOSet(TrackedSourceContext trackedSourceContext, int? batchSize = null, int? gapLimit = null, int? from = null) - { - var network = trackedSourceContext.Network; - var rpc = trackedSourceContext.RpcClient; - var derivationScheme = ((DerivationSchemeTrackedSource)trackedSourceContext.TrackedSource).DerivationStrategy; - if (!rpc.Capabilities.SupportScanUTXOSet) - throw new NBXplorerError(405, "scanutxoset-not-suported", "ScanUTXOSet is not supported for this currency").AsException(); - - ScanUTXOSetOptions options = new ScanUTXOSetOptions(); - if (batchSize != null) - options.BatchSize = batchSize.Value; - if (gapLimit != null) - options.GapLimit = gapLimit.Value; - if (from != null) - options.From = from.Value; - if (!ScanUTXOSetService.EnqueueScan(network, derivationScheme, options)) - throw new NBXplorerError(409, "scanutxoset-in-progress", "ScanUTXOSet has already been called for this derivationScheme").AsException(); - return Ok(); - } - - [HttpGet($"~/v1/{CommonRoutes.DerivationEndpoint}/utxos/scan")] - [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: typeof(DerivationSchemeTrackedSource))] - public IActionResult GetScanUTXOSetInformation(TrackedSourceContext trackedSourceContext) - { - var network = trackedSourceContext.Network; - var derivationScheme = ((DerivationSchemeTrackedSource)trackedSourceContext.TrackedSource).DerivationStrategy; - var info = ScanUTXOSetService.GetInformation(network, derivationScheme); - if (info == null) - throw new NBXplorerError(404, "scanutxoset-info-not-found", "ScanUTXOSet has not been called with this derivationScheme of the result has expired").AsException(); - return Json(info, network.Serializer.Settings); - } - [HttpPost($"~/v1/{CommonRoutes.DerivationEndpoint}/prune")] [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: [typeof(DerivationSchemeTrackedSource)])] public async Task Prune(TrackedSourceContext trackedSourceContext, [FromBody] PruneRequest request) diff --git a/NBXplorer/Controllers/GroupsController.cs b/NBXplorer/Controllers/GroupsController.cs index e13f84484..19e0a3ca2 100644 --- a/NBXplorer/Controllers/GroupsController.cs +++ b/NBXplorer/Controllers/GroupsController.cs @@ -155,6 +155,13 @@ await conn.ExecuteAsync(Repository.InsertScriptsScript + } return Ok(); } + [HttpGet($"{CommonRoutes.BaseCryptoEndpoint}/{CommonRoutes.GroupEndpoint}/addresses")] + public async Task GetGroupAddresses(TrackedSourceContext trackedSourceContext) + { + var addresses = await trackedSourceContext.Repository.GetAddresses(trackedSourceContext.TrackedSource, + trackedSourceContext.Network); + return Ok(addresses); + } private string GetWid(GroupChild c) { diff --git a/NBXplorer/ScanUTXOSetService.cs b/NBXplorer/ScanUTXOSetService.cs index d3626b0b2..844d2c76c 100644 --- a/NBXplorer/ScanUTXOSetService.cs +++ b/NBXplorer/ScanUTXOSetService.cs @@ -42,18 +42,18 @@ public class ScanUTXOSetService : IHostedService class ScanUTXOWorkItem { public ScanUTXOWorkItem(NBXplorerNetwork network, - DerivationStrategyBase derivationStrategy) + TrackedSource trackedSource) { Network = network; - DerivationStrategy = new DerivationSchemeTrackedSource(derivationStrategy); - Id = DerivationStrategy.ToString(); + TrackedSource = trackedSource; + Id = TrackedSource.ToString(); StartTime = DateTime.UtcNow; } public string Id { get; set; } public DateTimeOffset StartTime { get; set; } public ScanUTXOSetOptions Options { get; set; } public NBXplorerNetwork Network { get; } - public DerivationSchemeTrackedSource DerivationStrategy { get; set; } + public TrackedSource TrackedSource { get; set; } public ScanUTXOInformation State { get; set; } public bool Finished { get; internal set; } } @@ -77,9 +77,9 @@ public ScanUTXOSetService(ScanUTXOSetServiceAccessor accessor, Channel _Channel = Channel.CreateBounded(500); ConcurrentDictionary _Progress = new ConcurrentDictionary(); - internal bool EnqueueScan(NBXplorerNetwork network, DerivationStrategyBase derivationScheme, ScanUTXOSetOptions options) + internal bool EnqueueScan(NBXplorerNetwork network, TrackedSource trackedSource, ScanUTXOSetOptions options) { - var workItem = new ScanUTXOWorkItem(network, derivationScheme) + var workItem = new ScanUTXOWorkItem(network, trackedSource) { State = new ScanUTXOInformation() { @@ -138,7 +138,7 @@ private async Task Listen() Logs.Explorer.LogError($"{workItem.Network.CryptoCode}: Work has been scheduled for {item}, but the work has not been found in _Progress dictionary. This is likely a bug, contact NBXplorer developers."); continue; } - Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Start scanning {workItem.DerivationStrategy.ToPrettyString()} from index {workItem.Options.From} with gap limit {workItem.Options.GapLimit}, batch size {workItem.Options.BatchSize}"); + Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Start scanning {workItem.TrackedSource.ToPrettyString()} from index {workItem.Options.From} with gap limit {workItem.Options.GapLimit}, batch size {workItem.Options.BatchSize}"); var rpc = RpcClients.Get(workItem.Network); try { @@ -149,13 +149,25 @@ private async Task Listen() From = workItem.Options.From, StartedAt = DateTimeOffset.UtcNow }; - foreach (var feature in workItem.DerivationStrategy.GetDerivationFeatures(keyPathTemplates)) + switch (workItem.TrackedSource) { - workItem.State.Progress.HighestKeyIndexFound.Add(feature, null); + case AddressTrackedSource addressTrackedSource: + break; + case DerivationSchemeTrackedSource derivationSchemeTrackedSource: + foreach (var feature in derivationSchemeTrackedSource.GetDerivationFeatures(keyPathTemplates)) + { + workItem.State.Progress.HighestKeyIndexFound.Add(feature, null); + } + break; + case GroupTrackedSource groupTrackedSource: + break; + default: + throw new ArgumentOutOfRangeException(); } + workItem.State.Progress.UpdateRemainingBatches(workItem.Options.GapLimit); workItem.State.Status = ScanUTXOStatus.Pending; - var scannedItems = GetScannedItems(workItem, workItem.State.Progress, workItem.Network); + var scannedItems = await GetScannedItems(workItem, workItem.State.Progress, workItem.Network); var scanning = rpc.StartScanTxoutSetExAsync(new ScanTxoutSetParameters(scannedItems.Descriptors), _Cts.Token); while (true) @@ -186,8 +198,8 @@ private async Task Listen() progressObj.TotalSearched += scannedItems.Descriptors.Count; progressObj.UpdateRemainingBatches(workItem.Options.GapLimit); progressObj.UpdateOverallProgress(); - Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Scanning of batch {workItem.State.Progress.BatchNumber} for {workItem.DerivationStrategy.ToPrettyString()} complete with {outputs.Length} UTXOs fetched"); - await UpdateRepository(rpc, workItem.DerivationStrategy, repo, outputs, scannedItems, progressObj); + Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Scanning of batch {workItem.State.Progress.BatchNumber} for {workItem.TrackedSource.ToPrettyString()} complete with {outputs.Length} UTXOs fetched"); + await UpdateRepository(rpc, workItem.TrackedSource, repo, outputs, scannedItems, progressObj); if (progressObj.RemainingBatches <= -1) { @@ -201,12 +213,12 @@ private async Task Listen() progressObj.UpdateOverallProgress(); workItem.State.Progress = progressObj; workItem.State.Status = ScanUTXOStatus.Complete; - Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Scanning {workItem.DerivationStrategy.ToPrettyString()} complete {progressObj.Found} UTXOs found in total"); + Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Scanning {workItem.TrackedSource.ToPrettyString()} complete {progressObj.Found} UTXOs found in total"); break; } else { - scannedItems = GetScannedItems(workItem, progressObj, workItem.Network); + scannedItems = await GetScannedItems(workItem, progressObj, workItem.Network); workItem.State.Progress = progressObj; scanning = rpc.StartScanTxoutSetAsync(new ScanTxoutSetParameters(scannedItems.Descriptors)); } @@ -225,7 +237,7 @@ private async Task Listen() var progress = workItem.State.Progress.Clone(); progress.CompletedAt = DateTimeOffset.UtcNow; workItem.State.Progress = progress; - Logs.Explorer.LogError(ex, $"{workItem.Network.CryptoCode}: Error while scanning {workItem.DerivationStrategy.ToPrettyString()}"); + Logs.Explorer.LogError(ex, $"{workItem.Network.CryptoCode}: Error while scanning {workItem.TrackedSource.ToPrettyString()}"); } finally { @@ -241,7 +253,7 @@ private async Task Listen() } } - private async Task UpdateRepository(RPCClient client, DerivationSchemeTrackedSource trackedSource, Repository repo, ScanTxoutOutput[] outputs, ScannedItems scannedItems, ScanUTXOProgress progressObj) + private async Task UpdateRepository(RPCClient client, TrackedSource trackedSource, Repository repo, ScanTxoutOutput[] outputs, ScannedItems scannedItems, ScanUTXOProgress progressObj) { var blockHeaders = await client.GetBlockHeadersAsync(outputs.Select(o => o.Height).Distinct().ToList(), _Cts.Token); @@ -275,7 +287,8 @@ await repo.SaveKeyInformations(scannedItems. return false; return p.Index.Value <= highest.Value; }).ToArray()); - await repo.UpdateAddressPool(trackedSource, progressObj.HighestKeyIndexFound); + if(trackedSource is DerivationSchemeTrackedSource derivationSchemeTrackedSource) + await repo.UpdateAddressPool(derivationSchemeTrackedSource, progressObj.HighestKeyIndexFound); DateTimeOffset now = DateTimeOffset.UtcNow; var records = data.Select(d => SaveTransactionRecord.Create( @@ -287,42 +300,57 @@ await repo.SaveKeyInformations(scannedItems. await repo.SaveMatches(query, records.ToArray()); } - private ScannedItems GetScannedItems(ScanUTXOWorkItem workItem, ScanUTXOProgress progress, NBXplorerNetwork network) + private async Task GetScannedItems(ScanUTXOWorkItem workItem, ScanUTXOProgress progress, NBXplorerNetwork network) { var items = new ScannedItems(); - var derivationStrategy = workItem.DerivationStrategy; - foreach (var feature in derivationStrategy.GetDerivationFeatures(keyPathTemplates)) + + switch (workItem.TrackedSource) { - var lineDerivation = workItem.DerivationStrategy.DerivationStrategy.GetLineFor(keyPathTemplates, feature); - Enumerable.Range(progress.From, progress.Count) - .Select(index => - { - var keyPath = (lineDerivation as KeyPathTemplateDerivationLine)?.KeyPathTemplate.GetKeyPath(index, false); - var derivation = lineDerivation.Derive((uint)index); - var info = new KeyPathInformation() - { - ScriptPubKey = derivation.ScriptPubKey, - DerivationStrategy = derivationStrategy.DerivationStrategy, - Feature = feature, - KeyPath = keyPath, - Redeem = derivation.Redeem, - TrackedSource = derivationStrategy, - Address = derivation.ScriptPubKey.GetDestinationAddress(network.NBitcoinNetwork), - Index = index - }; - if (network.IsElement && !workItem.DerivationStrategy.DerivationStrategy.Unblinded()) - { - var blindingPubKey = - NBXplorer.NBXplorerNetworkProvider.LiquidNBXplorerNetwork - .GenerateBlindingKey(derivationStrategy.DerivationStrategy, keyPath, derivation.ScriptPubKey, network.NBitcoinNetwork).PubKey; - info.Address = new BitcoinBlindedAddress(blindingPubKey, info.Address); - } - items.Descriptors.Add(OutputDescriptor.NewRaw(info.ScriptPubKey, network.NBitcoinNetwork)); - items.KeyPathInformations.TryAdd(info.ScriptPubKey, info); - return info; - }).All(_ => true); + case AddressTrackedSource addressTrackedSource: + items.Descriptors.Add(OutputDescriptor.NewRaw(addressTrackedSource.ScriptPubKey, network.NBitcoinNetwork)); + break; + case DerivationSchemeTrackedSource derivationSchemeTrackedSource: + foreach (var feature in derivationSchemeTrackedSource.GetDerivationFeatures(keyPathTemplates)) + { + var lineDerivation = derivationSchemeTrackedSource.DerivationStrategy.GetLineFor(keyPathTemplates, feature); + Enumerable.Range(progress.From, progress.Count) + .Select(index => + { + var keyPath = (lineDerivation as KeyPathTemplateDerivationLine)?.KeyPathTemplate.GetKeyPath(index, false); + var derivation = lineDerivation.Derive((uint)index); + var info = new KeyPathInformation() + { + ScriptPubKey = derivation.ScriptPubKey, + DerivationStrategy = derivationSchemeTrackedSource.DerivationStrategy, + Feature = feature, + KeyPath = keyPath, + Redeem = derivation.Redeem, + TrackedSource = derivationSchemeTrackedSource, + Address = derivation.ScriptPubKey.GetDestinationAddress(network.NBitcoinNetwork), + Index = index + }; + if (network.IsElement && !derivationSchemeTrackedSource.DerivationStrategy.Unblinded()) + { + var blindingPubKey = + NBXplorer.NBXplorerNetworkProvider.LiquidNBXplorerNetwork + .GenerateBlindingKey(derivationSchemeTrackedSource.DerivationStrategy, keyPath, derivation.ScriptPubKey, network.NBitcoinNetwork).PubKey; + info.Address = new BitcoinBlindedAddress(blindingPubKey, info.Address); + } + items.Descriptors.Add(OutputDescriptor.NewRaw(info.ScriptPubKey, network.NBitcoinNetwork)); + items.KeyPathInformations.TryAdd(info.ScriptPubKey, info); + return info; + }).All(_ => true); + } + break; + case GroupTrackedSource groupTrackedSource: + var addresses = await Repositories.GetRepository(network).GetAddresses(groupTrackedSource, network); + items.Descriptors.AddRange(addresses.Select(s => OutputDescriptor.NewRaw(BitcoinAddress.Create(s, network.NBitcoinNetwork).ScriptPubKey, network.NBitcoinNetwork))); + break; + default: + throw new ArgumentOutOfRangeException(); } - Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Start scanning batch {progress.BatchNumber} of {workItem.DerivationStrategy.ToPrettyString()} from index {progress.From}"); + + Logs.Explorer.LogInformation($"{workItem.Network.CryptoCode}: Start scanning batch {progress.BatchNumber} of {workItem.TrackedSource.ToPrettyString()} from index {progress.From}"); return items; } @@ -333,9 +361,9 @@ public Task StopAsync(CancellationToken cancellationToken) return _Task; } - public ScanUTXOInformation GetInformation(NBXplorerNetwork network, DerivationStrategyBase derivationScheme) + public ScanUTXOInformation GetInformation(NBXplorerNetwork network, TrackedSource trackedSource) { - _Progress.TryGetValue(new ScanUTXOWorkItem(network, derivationScheme).Id, out var workItem); + _Progress.TryGetValue(new ScanUTXOWorkItem(network, trackedSource).Id, out var workItem); return workItem?.State; } } From a07421ae9f0b7a9a7b17027de58e7cc32a303d6f Mon Sep 17 00:00:00 2001 From: "Andrew Camilleri (Kukks)" Date: Mon, 25 Aug 2025 22:57:55 +0200 Subject: [PATCH 2/3] iterate and add tests --- NBXplorer.Client/ExplorerClient.cs | 6 ++ NBXplorer.Tests/UnitTest1.Groups.cs | 55 +++++++++++++++++++ .../Controllers/CommonRoutesController.cs | 8 +++ NBXplorer/Controllers/GroupsController.cs | 7 --- 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 4e0ba0581..378ff2713 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -353,6 +353,12 @@ public Task GetBalanceAsync(TrackedSource trackedSource, Can { return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/balance", cancellation); } + + public async Task GetAddresses(TrackedSource trackedSource, CancellationToken cancellation = default) + { + var addresses = await SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/addresses", cancellation); + return addresses.Select(s => BitcoinAddress.Create(s, Network.NBitcoinNetwork)).ToArray(); + } public async Task IsTrackedAsync(TrackedSource trackedSource, CancellationToken cancellation = default) { diff --git a/NBXplorer.Tests/UnitTest1.Groups.cs b/NBXplorer.Tests/UnitTest1.Groups.cs index 4b02dc93d..db77cd3a1 100644 --- a/NBXplorer.Tests/UnitTest1.Groups.cs +++ b/NBXplorer.Tests/UnitTest1.Groups.cs @@ -120,6 +120,61 @@ public async Task CanAliceAndBobShareWallet() balance = await tester.Client.GetBalanceAsync(gts); Assert.Equal(Money.Coins(1.0m + 1.2m), balance.Unconfirmed); } + + [Fact] + public async Task CanGetGroupAddresses() + { + using var tester = ServerTester.Create(); + var g = await tester.Client.CreateGroupAsync(); + var addresses = Enumerable.Range(0, 10).Select(_ => new Key().GetAddress(ScriptPubKeyType.TaprootBIP86, tester.Network).ToString()).ToArray(); + await tester.Client.AddGroupAddressAsync("BTC", g.GroupId, addresses); + + var groupAddresses = await tester.Client.GetAddresses(new GroupTrackedSource(g.GroupId)); + Assert.Equal(addresses.Length, groupAddresses.Length); + foreach (var a in addresses) + { + Assert.Contains(BitcoinAddress.Create(a, tester.Network), groupAddresses); + } + } + + + [Fact] + public async Task CanScanUTXOSetForGroups() + { + using var tester = ServerTester.Create(); + var g = await tester.Client.CreateGroupAsync(Cancel); + var newAddress = new Key().GetAddress(ScriptPubKeyType.TaprootBIP86, tester.Network); + await tester.Client.AddGroupAddressAsync("BTC", g.GroupId, [newAddress.ToString()], Cancel); + + var txid = await tester.SendToAddressAsync(newAddress, Money.Coins(1.0m)); + tester.RPC.Generate(1); + var block = await tester.RPC.GetBlockAsync(await tester.RPC.GetBestBlockHashAsync(Cancel), Cancel); + var match = block.Transactions.Single(t => t.GetHash() == txid); + var coin = match.Outputs.AsCoins().Single(c => c.ScriptPubKey == newAddress.ScriptPubKey); + + var gts = new GroupTrackedSource(g.GroupId); + await tester.Client.ScanUTXOSetAsync(gts, cancellation: Cancel); + + ScanUTXOInformation progress = null; + while (true) + { + progress = await tester.Client.GetScanUTXOSetInformationAsync(gts, Cancel); + Assert.NotNull(progress); + if (progress.Status is ScanUTXOStatus.Complete or ScanUTXOStatus.Error) + break; + await Task.Delay(100, Cancel); + } + Assert.Equal(ScanUTXOStatus.Complete, progress.Status); + Assert.Equal(1, progress.Progress.Found); + + var utxos = await tester.Client.GetUTXOsAsync(gts, Cancel); + var unspent = utxos.GetUnspentCoins(); + Assert.Single(unspent); + Assert.Equal(coin.Outpoint, unspent[0].Outpoint); + + var balance = await tester.Client.GetBalanceAsync(gts, Cancel); + Assert.Equal(Money.Coins(1.0m), balance.Confirmed); + } private async Task AssertNBXplorerException(int httpCode, Task task) { diff --git a/NBXplorer/Controllers/CommonRoutesController.cs b/NBXplorer/Controllers/CommonRoutesController.cs index 70d05a2b0..1bcdad69b 100644 --- a/NBXplorer/Controllers/CommonRoutesController.cs +++ b/NBXplorer/Controllers/CommonRoutesController.cs @@ -58,6 +58,14 @@ public IActionResult ScanUTXOSet(TrackedSourceContext trackedSourceContext, int? throw new NBXplorerError(409, "scanutxoset-in-progress", "ScanUTXOSet has already been called for this derivationScheme").AsException(); return Ok(); } + + [HttpGet($"addresses")] + public async Task GetAddresses(TrackedSourceContext trackedSourceContext) + { + var addresses = await trackedSourceContext.Repository.GetAddresses(trackedSourceContext.TrackedSource, + trackedSourceContext.Network); + return Ok(addresses); + } [HttpGet($"utxos/scan")] [TrackedSourceContext.TrackedSourceContextRequirement()] diff --git a/NBXplorer/Controllers/GroupsController.cs b/NBXplorer/Controllers/GroupsController.cs index 19e0a3ca2..e13f84484 100644 --- a/NBXplorer/Controllers/GroupsController.cs +++ b/NBXplorer/Controllers/GroupsController.cs @@ -155,13 +155,6 @@ await conn.ExecuteAsync(Repository.InsertScriptsScript + } return Ok(); } - [HttpGet($"{CommonRoutes.BaseCryptoEndpoint}/{CommonRoutes.GroupEndpoint}/addresses")] - public async Task GetGroupAddresses(TrackedSourceContext trackedSourceContext) - { - var addresses = await trackedSourceContext.Repository.GetAddresses(trackedSourceContext.TrackedSource, - trackedSourceContext.Network); - return Ok(addresses); - } private string GetWid(GroupChild c) { From 833a4ea8e0a6efd4a3fef09ea0340d2a025bc28a Mon Sep 17 00:00:00 2001 From: "Andrew Camilleri (Kukks)" Date: Mon, 25 Aug 2025 23:08:48 +0200 Subject: [PATCH 3/3] fix dict kpi --- NBXplorer/ScanUTXOSetService.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/NBXplorer/ScanUTXOSetService.cs b/NBXplorer/ScanUTXOSetService.cs index 844d2c76c..f71766a82 100644 --- a/NBXplorer/ScanUTXOSetService.cs +++ b/NBXplorer/ScanUTXOSetService.cs @@ -262,13 +262,16 @@ private async Task UpdateRepository(RPCClient client, TrackedSource trackedSourc .Select(o => (Coins: o.Select(c => c.Coin).ToList(), BlockHeader: blockHeaders.ByHeight.TryGet(o.First().Height), TxId: o.Select(c => c.Coin.Outpoint.Hash).FirstOrDefault(), - KeyPathInformations: o.Select(c => scannedItems.KeyPathInformations[c.Coin.ScriptPubKey]).ToList())) + KeyPathInformations: o + .Select(c => scannedItems.KeyPathInformations.TryGet(c.Coin.ScriptPubKey)) + .Where(information => information is not null) + .ToList())) .Where(o => o.BlockHeader != null) .Select(o => { foreach (var keyInfo in o.KeyPathInformations) { - var index = keyInfo.Index.Value; + var index = keyInfo!.Index!.Value; var highest = progressObj.HighestKeyIndexFound[keyInfo.Feature]; if (highest == null || index > highest.Value) {