Skip to content

Commit be29ddb

Browse files
tjayrushtjayrush
authored
Merge pull request #3434 from TrueBlocks/develop
Update to Version 2.5.0 - Much better concurrent code
2 parents 4e530b4 + 30e21a3 commit be29ddb

File tree

95 files changed

+5023
-1615
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

95 files changed

+5023
-1615
lines changed

‎docs/content/api/openapi.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ info:
77
license:
88
name: GPL 3.0
99
url: http://www.gnu.org/licenses/
10-
version: 2.2.0-release
10+
version: 2.5.0-release
1111
description: >
1212
1313
A REST layer over the TrueBlocks application. With `chifra daemon`, you can

‎src/apps/chifra/internal/blocks/handle_decache.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func (opts *BlocksOptions) HandleDecache() error {
2323

2424
ctx := context.Background()
2525
fetchData := func(modelChan chan types.Modeler[types.RawModeler], errorChan chan error) {
26-
if msg, err := decache.Decache(opts.Conn, itemsToRemove, silent, walk.Cache_Blocks); err != nil {
26+
if msg, err := decache.Decache(opts.Conn, itemsToRemove, silent, opts.getCacheType()); err != nil {
2727
errorChan <- err
2828
} else {
2929
s := types.SimpleMessage{
@@ -36,3 +36,13 @@ func (opts *BlocksOptions) HandleDecache() error {
3636
opts.Globals.NoHeader = true
3737
return output.StreamMany(ctx, fetchData, opts.Globals.OutputOpts())
3838
}
39+
40+
func (opts *BlocksOptions) getCacheType() walk.CacheType {
41+
cT := walk.Cache_Blocks
42+
if opts.Logs {
43+
cT = walk.Cache_Logs
44+
} else if opts.Traces {
45+
cT = walk.Cache_Traces
46+
}
47+
return cT
48+
}

‎src/apps/chifra/internal/blocks/handle_hashes.go

+53-50
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,14 @@ package blocksPkg
66

77
import (
88
"context"
9-
"errors"
9+
"fmt"
1010
"sort"
1111

1212
"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/identifiers"
1313
"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
1414
"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
1515
"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
1616
"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
17-
"github.com/ethereum/go-ethereum"
1817
)
1918

2019
func (opts *BlocksOptions) HandleHashes() error {
@@ -24,62 +23,66 @@ func (opts *BlocksOptions) HandleHashes() error {
2423

2524
ctx, cancel := context.WithCancel(context.Background())
2625
fetchData := func(modelChan chan types.Modeler[types.RawBlock], errorChan chan error) {
27-
// var cnt int
28-
var err error
29-
var appMap map[types.SimpleAppearance]*types.SimpleBlock[string]
30-
if appMap, _, err = identifiers.AsMap[types.SimpleBlock[string]](chain, opts.BlockIds); err != nil {
26+
if sliceOfMaps, cnt, err := identifiers.AsSliceOfMaps[types.SimpleBlock[string]](chain, opts.BlockIds); err != nil {
3127
errorChan <- err
3228
cancel()
33-
}
3429

35-
bar := logger.NewBar(logger.BarOptions{
36-
Type: logger.Expanding,
37-
Enabled: !opts.Globals.TestMode,
38-
Total: int64(len(appMap)),
39-
})
30+
} else if cnt == 0 {
31+
errorChan <- fmt.Errorf("no blocks found for the query")
32+
cancel()
33+
34+
} else {
35+
bar := logger.NewBar(logger.BarOptions{
36+
Enabled: !testMode && !utils.IsTerminal(),
37+
Total: int64(cnt),
38+
})
4039

41-
iterFunc := func(app types.SimpleAppearance, value *types.SimpleBlock[string]) error {
42-
bn := uint64(app.BlockNumber)
43-
if block, err := opts.Conn.GetBlockHeaderByNumber(bn); err != nil {
44-
errorChan <- err
45-
if errors.Is(err, ethereum.NotFound) {
46-
errorChan <- errors.New("uncles not found")
40+
for _, thisMap := range sliceOfMaps {
41+
thisMap := thisMap
42+
for app := range thisMap {
43+
thisMap[app] = new(types.SimpleBlock[string])
4744
}
48-
cancel()
49-
return nil
50-
} else {
51-
bar.Tick()
52-
*value = block
53-
}
54-
return nil
55-
}
5645

57-
iterErrorChan := make(chan error)
58-
iterCtx, iterCancel := context.WithCancel(context.Background())
59-
defer iterCancel()
60-
go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
61-
for err := range iterErrorChan {
62-
if !testMode || nErrors == 0 {
63-
errorChan <- err
64-
nErrors++
65-
}
66-
}
67-
bar.Finish(true)
46+
items := make([]*types.SimpleBlock[string], 0, len(thisMap))
47+
iterFunc := func(app types.SimpleAppearance, value *types.SimpleBlock[string]) error {
48+
bn := uint64(app.BlockNumber)
49+
if block, err := opts.Conn.GetBlockHeaderByNumber(bn); err != nil {
50+
delete(thisMap, app)
51+
return err
52+
} else {
53+
*value = block
54+
bar.Tick()
55+
}
56+
return nil
57+
}
6858

69-
items := make([]*types.SimpleBlock[string], 0, len(appMap))
70-
for _, item := range appMap {
71-
items = append(items, item)
72-
}
73-
sort.Slice(items, func(i, j int) bool {
74-
if items[i].BlockNumber == items[j].BlockNumber {
75-
return items[i].Hash.Hex() < items[j].Hash.Hex()
76-
}
77-
return items[i].BlockNumber < items[j].BlockNumber
78-
})
59+
iterErrorChan := make(chan error)
60+
iterCtx, iterCancel := context.WithCancel(context.Background())
61+
defer iterCancel()
62+
go utils.IterateOverMap(iterCtx, iterErrorChan, thisMap, iterFunc)
63+
for err := range iterErrorChan {
64+
if !testMode || nErrors == 0 {
65+
errorChan <- err
66+
nErrors++
67+
}
68+
}
7969

80-
for _, item := range items {
81-
item := item
82-
modelChan <- item
70+
for _, item := range thisMap {
71+
items = append(items, item)
72+
}
73+
sort.Slice(items, func(i, j int) bool {
74+
if items[i].BlockNumber == items[j].BlockNumber {
75+
return items[i].Hash.Hex() < items[j].Hash.Hex()
76+
}
77+
return items[i].BlockNumber < items[j].BlockNumber
78+
})
79+
80+
for _, item := range items {
81+
item := item
82+
modelChan <- item
83+
}
84+
}
85+
bar.Finish(true /* newLine */)
8386
}
8487
}
8588

‎src/apps/chifra/internal/blocks/handle_logs.go

+73-66
Original file line numberDiff line numberDiff line change
@@ -39,85 +39,92 @@ func (opts *BlocksOptions) HandleLogs() error {
3939

4040
ctx, cancel := context.WithCancel(context.Background())
4141
fetchData := func(modelChan chan types.Modeler[types.RawLog], errorChan chan error) {
42-
// var cnt int
43-
var err error
44-
var appMap map[types.SimpleAppearance]*types.SimpleTransaction
45-
if appMap, _, err = identifiers.AsMap[types.SimpleTransaction](chain, opts.BlockIds); err != nil {
42+
if sliceOfMaps, cnt, err := identifiers.AsSliceOfMaps[types.SimpleTransaction](chain, opts.BlockIds); err != nil {
4643
errorChan <- err
4744
cancel()
48-
}
4945

50-
bar := logger.NewBar(logger.BarOptions{
51-
Enabled: !opts.Globals.TestMode,
52-
Total: int64(len(appMap)),
53-
})
46+
} else if cnt == 0 {
47+
errorChan <- fmt.Errorf("no blocks found for the query")
48+
cancel()
5449

55-
iterFunc := func(app types.SimpleAppearance, value *types.SimpleTransaction) error {
56-
if value.Receipt == nil {
57-
value.Receipt = &types.SimpleReceipt{}
58-
}
50+
} else {
51+
bar := logger.NewBar(logger.BarOptions{
52+
Enabled: !testMode && !utils.IsTerminal(),
53+
Total: int64(cnt),
54+
})
55+
56+
for _, thisMap := range sliceOfMaps {
57+
thisMap := thisMap
58+
for app := range thisMap {
59+
thisMap[app] = new(types.SimpleTransaction)
60+
}
5961

60-
bn := uint64(app.BlockNumber)
61-
ts := opts.Conn.GetBlockTimestamp(bn)
62-
if logs, err := opts.Conn.GetLogsByNumber(bn, ts); err != nil {
63-
errorChan <- fmt.Errorf("block at %d returned an error: %w", bn, err)
64-
return nil
65-
66-
} else if len(logs) == 0 {
67-
errorChan <- fmt.Errorf("block at %d has no logs", bn)
68-
return nil
69-
70-
} else {
71-
l := make([]types.SimpleLog, 0, len(logs))
72-
for index := range logs {
73-
if opts.Articulate {
74-
if err = abiCache.ArticulateLog(&logs[index]); err != nil {
75-
errorChan <- err // continue even with an error
62+
iterFunc := func(app types.SimpleAppearance, value *types.SimpleTransaction) error {
63+
if value.Receipt == nil {
64+
value.Receipt = &types.SimpleReceipt{}
65+
}
66+
67+
bn := uint64(app.BlockNumber)
68+
ts := opts.Conn.GetBlockTimestamp(bn)
69+
if logs, err := opts.Conn.GetLogsByNumber(bn, ts); err != nil {
70+
delete(thisMap, app)
71+
return fmt.Errorf("block at %d returned an error: %w", bn, err)
72+
73+
} else if len(logs) == 0 {
74+
delete(thisMap, app)
75+
return fmt.Errorf("block at %d has no logs", bn)
76+
77+
} else {
78+
l := make([]types.SimpleLog, 0, len(logs))
79+
for index := range logs {
80+
if opts.Articulate {
81+
if err = abiCache.ArticulateLog(&logs[index]); err != nil {
82+
errorChan <- err // continue even with an error
83+
}
84+
}
85+
l = append(l, logs[index])
7686
}
87+
value.Receipt.Logs = append(value.Receipt.Logs, l...)
88+
bar.Tick()
89+
return nil
7790
}
78-
l = append(l, logs[index])
7991
}
80-
bar.Tick()
81-
value.Receipt.Logs = append(value.Receipt.Logs, l...)
82-
}
83-
return nil
84-
}
8592

86-
iterErrorChan := make(chan error)
87-
iterCtx, iterCancel := context.WithCancel(context.Background())
88-
defer iterCancel()
89-
go utils.IterateOverMap(iterCtx, iterErrorChan, appMap, iterFunc)
90-
for err := range iterErrorChan {
91-
if !testMode || nErrors == 0 {
92-
errorChan <- err
93-
// Reporting more than one error causes tests to fail because they
94-
// appear concurrently so sort differently
95-
nErrors++
96-
}
97-
}
98-
bar.Finish(true)
93+
iterErrorChan := make(chan error)
94+
iterCtx, iterCancel := context.WithCancel(context.Background())
95+
defer iterCancel()
96+
go utils.IterateOverMap(iterCtx, iterErrorChan, thisMap, iterFunc)
97+
for err := range iterErrorChan {
98+
if !testMode || nErrors == 0 {
99+
errorChan <- err
100+
nErrors++
101+
}
102+
}
99103

100-
items := make([]types.SimpleLog, 0, len(appMap))
101-
for _, tx := range appMap {
102-
tx := tx
103-
items = append(items, tx.Receipt.Logs...)
104-
}
105-
sort.Slice(items, func(i, j int) bool {
106-
if items[i].BlockNumber == items[j].BlockNumber {
107-
if items[i].TransactionIndex == items[j].TransactionIndex {
108-
return items[i].LogIndex < items[j].LogIndex
104+
items := make([]types.SimpleLog, 0, len(thisMap))
105+
for _, tx := range thisMap {
106+
tx := tx
107+
items = append(items, tx.Receipt.Logs...)
109108
}
110-
return items[i].TransactionIndex < items[j].TransactionIndex
111-
}
112-
return items[i].BlockNumber < items[j].BlockNumber
113-
})
109+
sort.Slice(items, func(i, j int) bool {
110+
if items[i].BlockNumber == items[j].BlockNumber {
111+
if items[i].TransactionIndex == items[j].TransactionIndex {
112+
return items[i].LogIndex < items[j].LogIndex
113+
}
114+
return items[i].TransactionIndex < items[j].TransactionIndex
115+
}
116+
return items[i].BlockNumber < items[j].BlockNumber
117+
})
114118

115-
for _, item := range items {
116-
item := item
117-
if !logFilter.PassesFilter(&item) {
118-
continue
119+
for _, item := range items {
120+
item := item
121+
if !logFilter.PassesFilter(&item) {
122+
continue
123+
}
124+
modelChan <- &item
125+
}
119126
}
120-
modelChan <- &item
127+
bar.Finish(true /* newLine */)
121128
}
122129
}
123130

0 commit comments

Comments
 (0)