perf(blockfrost): parallelize UTxO script hydration

backend/blockfrost/blockfrost.go

@@ -43,6 +43,7 @@ const (
 	cacheExpiry                   = 5 * time.Minute
 	maxBlockfrostResponseBytes    = 10 * 1024 * 1024
 	maxBlockfrostErrorSnippetSize = 512
+	maxConcurrentUtxoHydrations   = 8
 )
 
 // NewBlockFrostChainContext creates a new BlockFrost backend.
@@ -239,6 +240,7 @@ func (b *BlockFrostChainContext) Tip() (uint64, error) {
 func (b *BlockFrostChainContext) Utxos(address common.Address) ([]common.Utxo, error) {
 	const maxPages = 1000
 	var allUtxos []common.Utxo
+	resolver := newScriptRefResolver(b)
 
 	for page := 1; page <= maxPages+1; page++ {
 		path := fmt.Sprintf("/addresses/%s/utxos?page=%d", address.String(), page)
@@ -258,13 +260,11 @@ func (b *BlockFrostChainContext) Utxos(address common.Address) ([]common.Utxo, e
 			return nil, fmt.Errorf("UTxO pagination exceeded %d pages; results may be incomplete", maxPages)
 		}
 
-		for _, raw := range rawUtxos {
-			utxo, err := b.hydrateUtxo(raw, address)
-			if err != nil {
-				return nil, fmt.Errorf("failed to parse UTxO %s#%d: %w", raw.TxHash, raw.OutputIndex, err)
-			}
-			allUtxos = append(allUtxos, utxo)
+		utxos, err := b.hydrateUtxoPage(rawUtxos, address, resolver.resolve)
+		if err != nil {
+			return nil, err
 		}
+		allUtxos = append(allUtxos, utxos...)
 	}
 	return allUtxos, nil
 }
@@ -1066,6 +1066,58 @@ func (raw *bfAddressUTxO) toUtxo(address common.Address) (common.Utxo, error) {
 }
 
 func (b *BlockFrostChainContext) hydrateUtxo(raw bfAddressUTxO, address common.Address) (common.Utxo, error) {
+	return b.hydrateUtxoWithScriptResolver(raw, address, b.scriptRefByHash)
+}
+
+func (b *BlockFrostChainContext) hydrateUtxoPage(
+	rawUtxos []bfAddressUTxO,
+	address common.Address,
+	resolveScript func(string) (*common.ScriptRef, error),
+) ([]common.Utxo, error) {
+	utxos := make([]common.Utxo, len(rawUtxos))
+	errs := make([]error, len(rawUtxos))
+
+	workers := min(len(rawUtxos), maxConcurrentUtxoHydrations)
+	jobs := make(chan int)
+	var wg sync.WaitGroup
+	for range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for index := range jobs {
+				utxo, err := b.hydrateUtxoWithScriptResolver(rawUtxos[index], address, resolveScript)
+				if err != nil {
+					errs[index] = fmt.Errorf(
+						"failed to parse UTxO %s#%d: %w",
+						rawUtxos[index].TxHash,
+						rawUtxos[index].OutputIndex,
+						err,
+					)
+					continue
+				}
+				utxos[index] = utxo
+			}
+		}()
+	}
+	for index := range rawUtxos {
+		jobs <- index
+	}
+	close(jobs)
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+	return utxos, nil
+}
+
+func (b *BlockFrostChainContext) hydrateUtxoWithScriptResolver(
+	raw bfAddressUTxO,
+	address common.Address,
+	resolveScript func(string) (*common.ScriptRef, error),
+) (common.Utxo, error) {
 	utxo, err := raw.toUtxo(address)
 	if err != nil {
 		return common.Utxo{}, err
@@ -1082,7 +1134,7 @@ func (b *BlockFrostChainContext) hydrateUtxo(raw bfAddressUTxO, address common.A
 		output.DatumOption = datumOpt
 	}
 	if raw.ReferenceScriptHash != "" {
-		scriptRef, err := b.scriptRefByHash(raw.ReferenceScriptHash)
+		scriptRef, err := resolveScript(raw.ReferenceScriptHash)
 		if err != nil {
 			return common.Utxo{}, fmt.Errorf("failed to resolve reference script %s: %w", raw.ReferenceScriptHash, err)
 		}
@@ -1091,6 +1143,49 @@ func (b *BlockFrostChainContext) hydrateUtxo(raw bfAddressUTxO, address common.A
 	return utxo, nil
 }
 
+type scriptRefResolver struct {
+	context *BlockFrostChainContext
+
+	mu      sync.Mutex
+	entries map[string]*scriptRefResolveResult
+}
+
+type scriptRefResolveResult struct {
+	done      chan struct{}
+	scriptRef *common.ScriptRef
+	err       error
+}
+
+func newScriptRefResolver(context *BlockFrostChainContext) *scriptRefResolver {
+	return &scriptRefResolver{
+		context: context,
+		entries: make(map[string]*scriptRefResolveResult),
+	}
+}
+
+func (r *scriptRefResolver) resolve(hashHex string) (*common.ScriptRef, error) {
+	// Script hashes are hexadecimal, so normalize the key to coalesce case-only
+	// variants returned by upstream services.
+	key := strings.ToLower(hashHex)
+
+	r.mu.Lock()
+	entry, ok := r.entries[key]
+	if !ok {
+		entry = &scriptRefResolveResult{done: make(chan struct{})}
+		r.entries[key] = entry
+	}
+	r.mu.Unlock()
+
+	if ok {
+		<-entry.done
+		return entry.scriptRef, entry.err
+	}
+
+	entry.scriptRef, entry.err = r.context.scriptRefByHash(key)
+	close(entry.done)
+	return entry.scriptRef, entry.err
+}
+
 // inlineDatumOptionFromBlockfrost builds an inline datum option from BlockFrost's
 // inline_datum field, which is a CBOR-encoded datum serialized as a hex string.
 // The original CBOR bytes are preserved exactly (no JSON decode/re-encode

backend/blockfrost/blockfrost_test.go

@@ -11,7 +11,10 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/blinklabs-io/gouroboros/cbor"
 	"github.com/blinklabs-io/gouroboros/ledger/babbage"
@@ -85,6 +88,194 @@ func TestHydrateUtxoResolvesInlineDatumAndReferenceScript(t *testing.T) {
 	}
 }
 
+func TestUtxosHydratesDuplicateReferenceScriptsOnce(t *testing.T) {
+	addr := testAddress(t)
+	script := common.PlutusV2Script([]byte{0x01, 0x02})
+	scriptHashHex := hex.EncodeToString(script.Hash().Bytes())
+	var scriptRequests atomic.Int32
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v0/addresses/" + addr.String() + "/utxos":
+			if r.URL.Query().Get("page") == "1" {
+				_ = json.NewEncoder(w).Encode([]bfAddressUTxO{
+					{
+						TxHash:              strings.Repeat("a", 64),
+						OutputIndex:         0,
+						Address:             addr.String(),
+						Amount:              []bfAddressAmount{{Unit: "lovelace", Quantity: "1000000"}},
+						ReferenceScriptHash: scriptHashHex,
+					},
+					{
+						TxHash:              strings.Repeat("b", 64),
+						OutputIndex:         1,
+						Address:             addr.String(),
+						Amount:              []bfAddressAmount{{Unit: "lovelace", Quantity: "2000000"}},
+						ReferenceScriptHash: strings.ToUpper(scriptHashHex),
+					},
+				})
+				return
+			}
+			_ = json.NewEncoder(w).Encode([]bfAddressUTxO{})
+		case "/api/v0/scripts/" + scriptHashHex + "/cbor":
+			scriptRequests.Add(1)
+			_ = json.NewEncoder(w).Encode(map[string]string{"cbor": hex.EncodeToString(script)})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	ctx := NewBlockFrostChainContext(server.URL, 0, "")
+	utxos, err := ctx.Utxos(addr)
+	if err != nil {
+		t.Fatalf("Utxos: %v", err)
+	}
+	if len(utxos) != 2 {
+		t.Fatalf("got %d UTxOs, want 2", len(utxos))
+	}
+	if got := scriptRequests.Load(); got != 1 {
+		t.Fatalf("script requests = %d, want 1", got)
+	}
+}
+
+func TestUtxosHydratesReferenceScriptsConcurrentlyInResponseOrder(t *testing.T) {
+	addr := testAddress(t)
+	const txHash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
+	rawUtxos := make([]bfAddressUTxO, maxConcurrentUtxoHydrations+1)
+	scripts := make(map[string]common.PlutusV2Script, len(rawUtxos))
+	for index := range rawUtxos {
+		script := common.PlutusV2Script{0x01, byte(index)}
+		scriptHashHex := hex.EncodeToString(script.Hash().Bytes())
+		scripts[scriptHashHex] = script
+		rawUtxos[index] = bfAddressUTxO{
+			TxHash:              txHash,
+			OutputIndex:         index,
+			Address:             addr.String(),
+			Amount:              []bfAddressAmount{{Unit: "lovelace", Quantity: "1000000"}},
+			ReferenceScriptHash: scriptHashHex,
+		}
+	}
+
+	var active, maxActive atomic.Int32
+	started := make(chan struct{}, len(rawUtxos))
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	releaseScripts := func() { releaseOnce.Do(func() { close(release) }) }
+	t.Cleanup(releaseScripts)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v0/addresses/" + addr.String() + "/utxos":
+			if r.URL.Query().Get("page") == "1" {
+				_ = json.NewEncoder(w).Encode(rawUtxos)
+				return
+			}
+			_ = json.NewEncoder(w).Encode([]bfAddressUTxO{})
+			return
+		}
+
+		const scriptPrefix = "/api/v0/scripts/"
+		if !strings.HasPrefix(r.URL.Path, scriptPrefix) || !strings.HasSuffix(r.URL.Path, "/cbor") {
+			http.NotFound(w, r)
+			return
+		}
+		hashHex := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, scriptPrefix), "/cbor")
+		script, ok := scripts[hashHex]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+
+		current := active.Add(1)
+		defer active.Add(-1)
+		for {
+			observed := maxActive.Load()
+			if current <= observed || maxActive.CompareAndSwap(observed, current) {
+				break
+			}
+		}
+		started <- struct{}{}
+		<-release
+		_ = json.NewEncoder(w).Encode(map[string]string{"cbor": hex.EncodeToString(script)})
+	}))
+	defer server.Close()
+
+	ctx := NewBlockFrostChainContext(server.URL, 0, "")
+	type result struct {
+		utxos []common.Utxo
+		err   error
+	}
+	results := make(chan result, 1)
+	go func() {
+		utxos, err := ctx.Utxos(addr)
+		results <- result{utxos: utxos, err: err}
+	}()
+
+	for range maxConcurrentUtxoHydrations {
+		select {
+		case <-started:
+		case <-time.After(time.Second):
+			t.Fatal("timed out waiting for concurrent script requests")
+		}
+	}
+	if got := maxActive.Load(); got != maxConcurrentUtxoHydrations {
+		t.Fatalf("maximum concurrent script requests = %d, want %d", got, maxConcurrentUtxoHydrations)
+	}
+	releaseScripts()
+
+	res := <-results
+	if res.err != nil {
+		t.Fatalf("Utxos: %v", res.err)
+	}
+	if len(res.utxos) != len(rawUtxos) {
+		t.Fatalf("got %d UTxOs, want %d", len(res.utxos), len(rawUtxos))
+	}
+	for index, utxo := range res.utxos {
+		if got := utxo.Id.Index(); got != uint32(index) {
+			t.Fatalf("UTxO at index %d has output index %d, want %d", index, got, index)
+		}
+	}
+}
+
+func TestUtxosReferenceScriptFailureIncludesUtxoContext(t *testing.T) {
+	addr := testAddress(t)
+	const txHash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	const scriptHash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v0/addresses/" + addr.String() + "/utxos":
+			if r.URL.Query().Get("page") == "1" {
+				_ = json.NewEncoder(w).Encode([]bfAddressUTxO{{
+					TxHash:              txHash,
+					OutputIndex:         3,
+					Address:             addr.String(),
+					Amount:              []bfAddressAmount{{Unit: "lovelace", Quantity: "1000000"}},
+					ReferenceScriptHash: scriptHash,
+				}})
+				return
+			}
+			_ = json.NewEncoder(w).Encode([]bfAddressUTxO{})
+		case "/api/v0/scripts/" + scriptHash + "/cbor":
+			http.Error(w, "script unavailable", http.StatusInternalServerError)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer server.Close()
+
+	_, err := NewBlockFrostChainContext(server.URL, 0, "").Utxos(addr)
+	if err == nil {
+		t.Fatal("expected Utxos to fail")
+	}
+	want := "failed to parse UTxO " + txHash + "#3: failed to resolve reference script " + scriptHash
+	if !strings.Contains(err.Error(), want) {
+		t.Fatalf("error = %q, want context containing %q", err, want)
+	}
+}
+
 func TestUtxoByRefFillsMissingTxHashOnTxUtxosOutputs(t *testing.T) {
 	addr := testAddress(t)
 	const txHashHex = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"