Merge branch 'master' into protocol-23

Author: Shaptic

.github/actions/setup-go/action.yml

@@ -32,15 +32,15 @@ runs:
     run: echo 'PREFIX=${{ github.workflow }}-${{ github.job }}-${{ runner.os }}-${{ inputs.go-version }}-matrix(${{ join(matrix.*,'|') }})' >> $GITHUB_ENV
 
   # Cache the Go Modules downloaded during the job.
-  - uses: actions/cache@v2
+  - uses: actions/cache@v4
     with:
       path: ~/go/pkg/mod
       key: ${{ env.PREFIX }}-go-mod-${{ hashFiles('**/go.sum') }}
       restore-keys: ${{ env.PREFIX }}-go-mod-
 
   # Cache any build and test artifacts during the job, which will speed up
   # rebuilds and cause test runs to skip tests that have no reason to rerun.
-  - uses: actions/cache@v2
+  - uses: actions/cache@v4
     with:
       path: ~/.cache/go-build
       key: ${{ env.PREFIX }}-go-build-${{ github.ref }}-${{ hashFiles('**', '!.git') }}

.github/workflows/horizon.yml

@@ -103,7 +103,7 @@ jobs:
 
       - name: Restore Horizon binary and integration tests source hash to cache
         id: horizon_binary_tests_hash
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ./empty
           lookup-only: true
@@ -114,7 +114,7 @@ jobs:
 
       - name: Save Horizon binary and integration tests source hash to cache
         if: ${{ success() && steps.horizon_binary_tests_hash.outputs.cache-hit != 'true' }}
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         with:
           path: ./empty
           key: ${{ env.COMBINED_SOURCE_HASH }}

.gitignore

@@ -21,3 +21,4 @@ debug
 *.db
 *.conf
 *.lock
+.proto_checksums

Makefile

@@ -73,3 +73,48 @@ xdr-clean:
 	rm $(DOWNLOADABLE_XDRS) || true
 
 xdr-update: xdr-clean xdr
+
+
+##############
+PROTO_DIR := protos
+GEN_SUFFIX := .pb.go
+PROTO_FILES := $(shell find $(PROTO_DIR) -name "*.proto")
+PROTO_CHECKSUM := .proto_checksums
+
+generate-proto:
+	@echo "Regenerating proto files..."
+	@touch $(PROTO_CHECKSUM)
+	@current_checksum=$$(sha256sum $(PROTO_FILES) | sha256sum | awk '{print $$1}'); \
+	stored_checksum=$$(cat $(PROTO_CHECKSUM)); \
+	if [ "$${current_checksum}" != "$${stored_checksum}" ]; then \
+    	echo "Changes detected. Regenerating all proto files..."; \
+    	MAP_OPTS=$$(for file in $(PROTO_FILES); do \
+    		rel_path=$$(echo $$file | sed 's|$(PROTO_DIR)/||'); \
+    		pkg_path=$$(dirname $$rel_path); \
+    		go_pkg="github.com/stellar/go/$$pkg_path"; \
+    		printf "M%s=%s," "$$rel_path" "$$go_pkg"; \
+    	done); \
+    	MAP_OPTS=$${MAP_OPTS%,}; \
+    	echo "Running protoc with options:"; \
+    	echo "  --go_out=."; \
+    	echo "  --go_opt=paths=source_relative"; \
+    	echo "  --go_opt=$$MAP_OPTS"; \
+    	echo "Proto Files:  $(PROTO_FILES)"; \
+    	protoc -I=$(PROTO_DIR) \
+    	       --go_out=. --go_opt=paths=source_relative \
+    	       --go_opt=$$MAP_OPTS \
+    	       $(PROTO_FILES); \
+    	echo "$${current_checksum}" > $(PROTO_CHECKSUM); \
+    else \
+    	echo "No changes detected in proto files."; \
+    fi
+
+
+regenerate-proto: $(PROTO_CHECKSUM)
+	rm -f $(PROTO_CHECKSUM)
+	$(MAKE) generate-proto
+
+$(PROTO_CHECKSUM):
+	@touch $(PROTO_CHECKSUM)
+
+.PHONY: generate-proto regenerate-proto

README.md

@@ -45,6 +45,7 @@ In addition to the other top-level packages, there are a few special directories
 * **support/scripts** contains single-file go programs and bash scripts used to support the development of this repo. 
 * **services** contains packages that compile to applications that are long-running processes (such as API servers).
 * **tools** contains packages that compile to command line applications.
+* **protos** contains the protobuf definitions for data-models that we want to generate for use with external stakeholders. Refer to [protos/README.md](./protos/README.md) for more information about code structure in `protos` directory
 
 Each of these directories have their own README file that explain further the nature of their contents.
 

go.mod

@@ -178,7 +178,7 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/grpc v1.64.1 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.34.2
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

historyarchive/archive.go

@@ -79,8 +79,8 @@ type ArchiveInterface interface {
 	ListAllBuckets() (chan string, chan error)
 	ListAllBucketHashes() (chan Hash, chan error)
 	ListCategoryCheckpoints(cat string, pth string) (chan uint32, chan error)
-	GetXdrStreamForHash(hash Hash) (*XdrStream, error)
-	GetXdrStream(pth string) (*XdrStream, error)
+	GetXdrStreamForHash(hash Hash) (*xdr.Stream, error)
+	GetXdrStream(pth string) (*xdr.Stream, error)
 	GetCheckpointManager() CheckpointManager
 	GetStats() []ArchiveStats
 }
@@ -391,19 +391,19 @@ func (a *Archive) GetBucketPathForHash(hash Hash) string {
 	)
 }
 
-func (a *Archive) GetXdrStreamForHash(hash Hash) (*XdrStream, error) {
+func (a *Archive) GetXdrStreamForHash(hash Hash) (*xdr.Stream, error) {
 	return a.GetXdrStream(a.GetBucketPathForHash(hash))
 }
 
-func (a *Archive) GetXdrStream(pth string) (*XdrStream, error) {
+func (a *Archive) GetXdrStream(pth string) (*xdr.Stream, error) {
 	if !strings.HasSuffix(pth, ".xdr.gz") {
 		return nil, errors.New("File has non-.xdr.gz suffix: " + pth)
 	}
 	rdr, err := a.cachedGet(pth)
 	if err != nil {
 		return nil, err
 	}
-	return NewXdrGzStream(rdr)
+	return xdr.NewGzStream(rdr)
 }
 
 func (a *Archive) cachedGet(pth string) (io.ReadCloser, error) {

historyarchive/archive_pool.go

@@ -226,17 +226,17 @@ func (pa *ArchivePool) PutRootHAS(has HistoryArchiveState, opts *CommandOptions)
 	})
 }
 
-func (pa *ArchivePool) GetXdrStreamForHash(hash Hash) (*XdrStream, error) {
-	var stream *XdrStream
+func (pa *ArchivePool) GetXdrStreamForHash(hash Hash) (*xdr.Stream, error) {
+	var stream *xdr.Stream
 	return stream, pa.runRoundRobin(func(ai ArchiveInterface) error {
 		var err error
 		stream, err = ai.GetXdrStreamForHash(hash)
 		return err
 	})
 }
 
-func (pa *ArchivePool) GetXdrStream(pth string) (*XdrStream, error) {
-	var stream *XdrStream
+func (pa *ArchivePool) GetXdrStream(pth string) (*xdr.Stream, error) {
+	var stream *xdr.Stream
 	return stream, pa.runRoundRobin(func(ai ArchiveInterface) error {
 		var err error
 		stream, err = ai.GetXdrStream(pth)

historyarchive/hash.go

@@ -8,9 +8,11 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+
+	"github.com/stellar/go/xdr"
 )
 
-type Hash [sha256.Size]byte
+type Hash xdr.Hash
 
 func DecodeHash(s string) (Hash, error) {
 	var h Hash

historyarchive/json.go

@@ -35,7 +35,7 @@ func DumpXdrAsJson(args []string) error {
 		}
 
 		base := path.Base(arg)
-		xr := NewXdrStream(rdr)
+		xr := xdr.NewStream(rdr)
 		n := 0
 		for {
 			var lhe xdr.LedgerHeaderHistoryEntry

historyarchive/mocks.go

@@ -1,8 +1,9 @@
 package historyarchive
 
 import (
-	"github.com/stellar/go/xdr"
 	"github.com/stretchr/testify/mock"
+
+	"github.com/stellar/go/xdr"
 )
 
 // MockArchive is a mockable archive.
@@ -99,14 +100,14 @@ func (m *MockArchive) ListCategoryCheckpoints(cat string, pth string) (chan uint
 	return make(chan uint32), make(chan error)
 }
 
-func (m *MockArchive) GetXdrStreamForHash(hash Hash) (*XdrStream, error) {
+func (m *MockArchive) GetXdrStreamForHash(hash Hash) (*xdr.Stream, error) {
 	a := m.Called(hash)
-	return a.Get(0).(*XdrStream), a.Error(1)
+	return a.Get(0).(*xdr.Stream), a.Error(1)
 }
 
-func (m *MockArchive) GetXdrStream(pth string) (*XdrStream, error) {
+func (m *MockArchive) GetXdrStream(pth string) (*xdr.Stream, error) {
 	a := m.Called(pth)
-	return a.Get(0).(*XdrStream), a.Error(1)
+	return a.Get(0).(*xdr.Stream), a.Error(1)
 }
 
 func (m *MockArchive) GetStats() []ArchiveStats {

historyarchive/verify.go

@@ -51,11 +51,11 @@ func SortTxsForHash(txset *xdr.TransactionSet) error {
 		hsh: make([]Hash, len(txset.Txs)),
 	}
 	for i, tx := range txset.Txs {
-		h, err := HashXdr(&tx)
+		h, err := xdr.HashXdr(&tx)
 		if err != nil {
 			return err
 		}
-		bh.hsh[i] = h
+		bh.hsh[i] = Hash(h)
 	}
 	sort.Sort(bh)
 	return nil
@@ -86,18 +86,18 @@ func HashEmptyTxSet(previousLedgerHash Hash) Hash {
 }
 
 func (arch *Archive) VerifyLedgerHeaderHistoryEntry(entry *xdr.LedgerHeaderHistoryEntry) error {
-	h, err := HashXdr(&entry.Header)
+	h, err := xdr.HashXdr(&entry.Header)
 	if err != nil {
 		return err
 	}
-	if h != Hash(entry.Hash) {
+	if h != entry.Hash {
 		return fmt.Errorf("Ledger %d expected hash %s, got %s",
 			entry.Header.LedgerSeq, Hash(entry.Hash), Hash(h))
 	}
 	arch.mutex.Lock()
 	defer arch.mutex.Unlock()
 	seq := uint32(entry.Header.LedgerSeq)
-	arch.actualLedgerHashes[seq] = h
+	arch.actualLedgerHashes[seq] = Hash(h)
 	arch.expectLedgerHashes[seq-1] = Hash(entry.Header.PreviousLedgerHash)
 	arch.expectTxSetHashes[seq] = Hash(entry.Header.ScpValue.TxSetHash)
 	arch.expectTxResultSetHashes[seq] = Hash(entry.Header.TxSetResultHash)
@@ -117,13 +117,13 @@ func (arch *Archive) VerifyTransactionHistoryEntry(entry *xdr.TransactionHistory
 }
 
 func (arch *Archive) VerifyTransactionHistoryResultEntry(entry *xdr.TransactionHistoryResultEntry) error {
-	h, err := HashXdr(&entry.TxResultSet)
+	h, err := xdr.HashXdr(&entry.TxResultSet)
 	if err != nil {
 		return err
 	}
 	arch.mutex.Lock()
 	defer arch.mutex.Unlock()
-	arch.actualTxResultSetHashes[uint32(entry.LedgerSeq)] = h
+	arch.actualTxResultSetHashes[uint32(entry.LedgerSeq)] = Hash(h)
 	return nil
 }