From 89140e50b3fcb681535a84c92abcb30ac4c676f7 Mon Sep 17 00:00:00 2001 From: Shreyan Gupta Date: Fri, 24 Jan 2025 04:05:07 +0530 Subject: [PATCH 1/3] [spellcheck] Part 3: Spell check directories debug_scripts and docs --- cspell.json | 184 +++++++++++++++++- debug_scripts/READEME.md | 8 +- debug_scripts/estimate_epoch_start_time.py | 143 +++++++------- debug_scripts/request_chain_info.py | 70 +++---- debug_scripts/send_validator_logs.py | 58 +++--- .../tests/send_validator_logs_test.py | 15 +- docs/advanced_configuration/networking.md | 1 - docs/architecture/README.md | 2 +- docs/architecture/gas/estimator.md | 1 + docs/architecture/gas/gas_profile.md | 1 + docs/architecture/how/README.md | 27 ++- docs/architecture/how/cross-shard.md | 6 +- docs/architecture/how/gas.md | 15 +- docs/architecture/how/gc.md | 2 +- docs/architecture/how/meta-tx.md | 8 +- docs/architecture/how/proofs.md | 5 +- docs/architecture/how/receipt-congestion.md | 8 +- docs/architecture/how/resharding_v2.md | 93 ++++----- docs/architecture/how/serialization.md | 9 +- docs/architecture/how/sync.md | 12 +- docs/architecture/how/tx_receipts.md | 3 +- docs/architecture/how/tx_routing.md | 2 +- docs/architecture/network.md | 29 ++- docs/architecture/next/README.md | 1 + .../next/catchup_and_state_sync.md | 19 +- .../malicious_chunk_producer_and_phase2.md | 63 +++--- docs/architecture/storage.md | 2 +- docs/architecture/storage/README.md | 15 +- docs/architecture/storage/database.md | 8 +- docs/architecture/storage/flat_storage.md | 18 +- docs/architecture/storage/flow.md | 1 + docs/architecture/storage/primitives.md | 2 +- docs/architecture/storage/trie_storage.md | 4 +- docs/misc/archival_data_recovery.md | 8 +- docs/misc/contract_distribution.md | 13 +- docs/misc/state_sync_dump.md | 6 + docs/misc/state_sync_from_external_storage.md | 7 +- docs/misc/state_witness_size_limits.md | 30 +-- docs/practices/docs.md | 2 +- .../engaging_and_effective_communication.md | 23 ++- docs/practices/fast_builds.md | 15 +- docs/practices/protocol_upgrade.md | 43 ++-- docs/practices/rust.md | 1 + docs/practices/security_vulnerabilities.md | 10 +- docs/practices/style.md | 32 +-- docs/practices/testing/coverage.md | 3 +- docs/practices/testing/test_utils.md | 3 +- .../when_to_use_private_repository.md | 12 +- docs/practices/workflows/README.md | 2 +- docs/practices/workflows/deploy_a_contract.md | 13 +- docs/practices/workflows/gas_estimations.md | 1 + docs/practices/workflows/io_trace.md | 14 +- .../workflows/localnet_on_many_machines.md | 16 +- docs/practices/workflows/otel_traces.md | 7 +- docs/practices/workflows/profiling.md | 34 ++-- docs/practices/workflows/run_a_node.md | 8 +- docs/test_networks/mainnet_spoon.md | 22 +-- 57 files changed, 687 insertions(+), 473 deletions(-) diff --git a/cspell.json b/cspell.json index 226615abf4a..f16feb54fcb 100644 --- a/cspell.json +++ b/cspell.json @@ -1,66 +1,183 @@ { "version": "0.2", - "ignorePaths": [], + "ignorePaths": [ + "*.ros", + "benchmarks/continuous/db/tool/orm/migrations/**", + "chain/rosetta-rpc/src/adapters/snapshots/**", + "core/primitives/res/epoch_configs/**.json", + "cspell.json", + "debug_scripts/Pipfile", + "docs/images/**", + ], "dictionaryDefinitions": [], "dictionaries": [], "words": [ "acks", "actix", + "addrs", + "AFAIU", + "akhi", "Aleksandr", "allocs", + "alot", + "appender", "archivals", + "autofail", "backpressure", "Banasik", + "BASEPOINT", + "benchmarknet", + "betanet", + "bijective", + "bitmask", "bitvec", + "BLOCKLIST", + "bootnodes", "borsh", "bufbuild", "bytesize", + "calimero", + "chacha", + "Checkpointing", "chrono", "Chudas", - "cloneable", "Clippy", + "cloneable", + "clusterfuzz", + "codegen", + "colddb", + "Condvar", + "Condvars", + "conv", + "cpus", + "cranelift", + "creds", + "dalek", "datas", + "dbprofile", "dealloc", "deallocate", "deallocating", "deallocation", + "Demultiplexer", "demultiplexing", + "Demux", + "demuxed", "Deque", + "DEVNOTE", "Diop", + "distro", + "doctest", "doctests", + "dont", + "dontcare", "doomslug", + "doomslugs", + "fastforward", + "finalizable", + "flamegraph", + "flatstore", "fmtr", "forknet", "freelist", "freelists", + "gced", + "Ggas", + "gprusak", + "hashlib", + "idempotently", + "illia", + "insta", + "Interruptible", "itertools", "itoa", + "jbajic", + "Justfile", + "kaiching", + "Kbps", + "keccak", + "keypair", + "keypairs", "kickout", "Kickouts", + "kkuuue", + "libfuzzer", + "librocksdb", + "lightclient", + "localnet", "Logunov", + "Mbps", + "memtable", "memtrie", "memtries", + "merkelization", "Merkle", "merklize", + "merklized", + "merklizing", "metadatas", + "microarchitectural", + "MILLI", + "Millinear", + "millis", + "mixeddb", "mocknet", + "multiexp", "multinode", + "multizip", + "nagisa", + "nanos", + "nanosec", + "nayduck", "nearcore", + "nearcrowd", "neard", + "nearlib", + "nearone", + "nearprotocol", + "nearup", "nextest", + "nikurt", + "NOFILE", + "Nomicon", + "nonexpired", + "nonleaf", + "numer", "nums", "oneshot", + "ords", + "peekable", + "peermanager", + "perc", + "petagas", + "PGAS", + "pinnable", + "pipenv", + "postprocesses", "Prefetcher", "prefetchers", + "prefunded", + "privkey", "profraw", "profraws", + "pugachag", "pytest", "pytests", + "PYTHONPATH", + "QUIC", + "RAII", + "readwrite", + "rebalance", + "rebalances", + "rebalancing", + "recoverydb", "refcount", "refcounted", "refcounting", "reindexing", + "replayability", "repr", + "reqwest", "reshard", "resharded", "Resharder", @@ -68,34 +185,95 @@ "reshardings", "respawn", "respawns", + "restake", + "retryable", + "ripemd", + "RISTRETTO", + "rlimit", + "rmem", + "rngs", "rocksdb", + "rosettatest", + "rosettatesting", + "routeback", + "rsplit", + "rtype", "RUSTC", + "rustdoc", "RUSTFLAGS", - "rtype", + "rustup", + "Ryzen", + "saketh", + "Satoshi", + "SECP", + "Seedable", "serde", "Shreyan", + "smallvec", + "spammy", + "splitdb", "Spurio", "stdx", "structs", + "subaccount", + "subcmd", + "subhandler", + "subpaths", + "subrequest", + "subscope", + "subsec", + "subservices", "subslice", + "syscall", + "sysinfo", "taiki", + "telezhnaya", "tempfile", "TERA", + "teragas", + "testchain", + "testdb", "testlib", "testloop", "testonly", "TGAS", "thiserror", + "threadpool", + "trisfald", + "txns", + "typenum", "udeps", "uids", + "uncompiled", + "uncongested", "unittests", + "unlabel", + "unorphaned", "unstake", + "unstaked", + "unstakes", + "unstaking", "unsync", + "upto", + "usize", + "valgrind", + "valset", + "Vecs", + "virtualenv", + "vnet", "wacban", "Waclaw", + "walltime", "wasmer", "wasms", + "Wasmtime", + "webrtc", + "wmem", + "xoraddr", + "xorshift", + "yansi", "yocto", + "yoctonear", "zstd", "Zulip" ], diff --git a/debug_scripts/READEME.md b/debug_scripts/READEME.md index 7c191759914..bd2eb5872ff 100644 --- a/debug_scripts/READEME.md +++ b/debug_scripts/READEME.md @@ -1,14 +1,14 @@ # Debug Scripts + ## Content + * request_chain_info.py This script can be used to request blockchain info * send_validator_logs.py - This script can be used to send Validator logs to a Pagoda S3 bucket when issues are encountered. The pagoda team can use the logs to help the validators troubleshoot issues. - ## Instruction to RUN ``` @@ -21,7 +21,9 @@ ``` ## Instruction to run test + Add nearcore/debug_scripts to your PYTHONPATH + ``` export PYTHONPATH="/nearcore/debug_scripts:$PYTHONPATH" ``` @@ -32,5 +34,3 @@ python3 -m pipenv sync python3 -m pipenv shell python3 -m unittest tests.send_validator_logs_test ``` - - diff --git a/debug_scripts/estimate_epoch_start_time.py b/debug_scripts/estimate_epoch_start_time.py index b5bb568a6cd..d70e56632de 100644 --- a/debug_scripts/estimate_epoch_start_time.py +++ b/debug_scripts/estimate_epoch_start_time.py @@ -14,14 +14,12 @@ def get_block(url, block_hash): "method": "block", } - payload["params"] = { - "block_id": block_hash - } if block_hash is not None else { - "finality": "final" - } + payload["params"] = ( + {"block_id": block_hash} if block_hash is not None else {"finality": "final"} + ) response = requests.post(url, json=payload) - return response.json()['result']['header'] + return response.json()["result"]["header"] def ns_to_seconds(ns): @@ -29,14 +27,14 @@ def ns_to_seconds(ns): def format_time(seconds): + # cspell:words gmtime return time.strftime("%H hours, %M minutes", time.gmtime(seconds)) # Function to fetch epoch lengths for the past n epochs and calculate the weighted average using exponential decay -def get_exponential_weighted_epoch_lengths(url, - starting_block_hash, - num_epochs, - decay_rate=0.1): +def get_exponential_weighted_epoch_lengths( + url, starting_block_hash, num_epochs, decay_rate=0.1 +): epoch_lengths = [] current_hash = starting_block_hash @@ -45,14 +43,14 @@ def get_exponential_weighted_epoch_lengths(url, block_data = get_block(url, current_hash) # Get the timestamp of this block (start of current epoch) - current_timestamp = int(block_data['timestamp']) + current_timestamp = int(block_data["timestamp"]) # Get the next epoch hash (last block hash of previous epoch.) - previous_hash = block_data['next_epoch_id'] + previous_hash = block_data["next_epoch_id"] # Fetch the block data for start of previous epoch previous_block_data = get_block(url, previous_hash) - previous_timestamp = int(previous_block_data['timestamp']) + previous_timestamp = int(previous_block_data["timestamp"]) # Calculate the length of the epoch in nanoseconds epoch_length = current_timestamp - previous_timestamp @@ -90,11 +88,13 @@ def valid_timezone(timezone_str): # Function to approximate future epoch start dates -def predict_future_epochs(starting_epoch_timestamp, avg_epoch_length, - num_future_epochs, target_timezone): +def predict_future_epochs( + starting_epoch_timestamp, avg_epoch_length, num_future_epochs, target_timezone +): future_epochs = [] current_timestamp = ns_to_seconds( - starting_epoch_timestamp) # Convert from nanoseconds to seconds + starting_epoch_timestamp + ) # Convert from nanoseconds to seconds for i in range(1, num_future_epochs + 1): # Add the average epoch length for each future epoch @@ -102,18 +102,17 @@ def predict_future_epochs(starting_epoch_timestamp, avg_epoch_length, future_epochs.append(future_timestamp) # Convert timestamp to datetime in target timezone - future_datetime = datetime.fromtimestamp(future_timestamp, - target_timezone) + future_datetime = datetime.fromtimestamp(future_timestamp, target_timezone) # Format date - future_date = future_datetime.strftime('%Y-%m-%d %H:%M:%S %Z%z %A') + future_date = future_datetime.strftime("%Y-%m-%d %H:%M:%S %Z%z %A") print(f"Predicted start of epoch {i}: {future_date}") return future_epochs def find_epoch_for_timestamp(future_epochs, voting_timestamp): - for (epoch_number, epoch_timestamp) in enumerate(future_epochs): + for epoch_number, epoch_timestamp in enumerate(future_epochs): if voting_timestamp < epoch_timestamp: return epoch_number return len(future_epochs) @@ -129,7 +128,8 @@ def find_best_voting_hour(voting_date_str, future_epochs): for hour in range(24): # Construct datetime for each hour of the voting date voting_datetime = datetime.strptime( - f"{voting_date_str} {hour:02d}:00:00", '%Y-%m-%d %H:%M:%S') + f"{voting_date_str} {hour:02d}:00:00", "%Y-%m-%d %H:%M:%S" + ) voting_datetime = pytz.utc.localize(voting_datetime) voting_timestamp = voting_datetime.timestamp() @@ -146,10 +146,8 @@ def find_best_voting_hour(voting_date_str, future_epochs): ) break - protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - - 1] - protocol_upgrade_datetime = datetime.fromtimestamp( - protocol_upgrade_timestamp) + protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - 1] + protocol_upgrade_datetime = datetime.fromtimestamp(protocol_upgrade_timestamp) upgrade_hour_utc = protocol_upgrade_datetime.hour if WORKING_HOURS_START <= upgrade_hour_utc < WORKING_HOURS_END: @@ -159,7 +157,7 @@ def find_best_voting_hour(voting_date_str, future_epochs): print( f"\nVoting hours on {voting_date_str} UTC that result in upgrade during working hours (UTC {WORKING_HOURS_START}:00-{WORKING_HOURS_END}:00):" ) - for (hour, epoch) in valid_hours: + for hour, epoch in valid_hours: print(f"- {hour:02d}:00, Upgrade Epoch: {epoch}") else: print( @@ -169,7 +167,7 @@ def find_best_voting_hour(voting_date_str, future_epochs): def valid_voting_datetime(s): try: - dt = datetime.strptime(s, '%Y-%m-%d %H:%M:%S') + dt = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") return dt except ValueError: raise argparse.ArgumentTypeError( @@ -181,8 +179,7 @@ def find_protocol_upgrade_time(voting_date, future_epochs, target_timezone): voting_datetime = target_timezone.localize(voting_date) # Find the epoch T in which the voting date falls - epoch_T = find_epoch_for_timestamp(future_epochs, - voting_datetime.timestamp()) + epoch_T = find_epoch_for_timestamp(future_epochs, voting_datetime.timestamp()) if epoch_T <= 0: print("Error: Voting date is before the first predicted epoch.") return @@ -194,12 +191,13 @@ def find_protocol_upgrade_time(voting_date, future_epochs, target_timezone): "Not enough future epochs predicted to determine the protocol upgrade time." ) return - protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - - 1] + protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - 1] protocol_upgrade_datetime = datetime.fromtimestamp( - protocol_upgrade_timestamp, tz=target_timezone) + protocol_upgrade_timestamp, tz=target_timezone + ) protocol_upgrade_formatted = protocol_upgrade_datetime.strftime( - '%Y-%m-%d %H:%M:%S %Z%z %A') + "%Y-%m-%d %H:%M:%S %Z%z %A" + ) print(f"\nVoting date falls into epoch {epoch_T}.") print( f"Protocol upgrade will happen at the start of epoch {protocol_upgrade_epoch_number}: {protocol_upgrade_formatted}" @@ -209,23 +207,29 @@ def find_protocol_upgrade_time(voting_date, future_epochs, target_timezone): # Main function to run the process def main(args): latest_block = get_block(args.url, None) - next_epoch_id = latest_block['next_epoch_id'] + next_epoch_id = latest_block["next_epoch_id"] current_epoch_first_block = get_block(args.url, next_epoch_id) - current_timestamp = int(current_epoch_first_block['timestamp'] - ) # Current epoch start timestamp in nanoseconds + current_timestamp = int( + current_epoch_first_block["timestamp"] + ) # Current epoch start timestamp in nanoseconds # Get epoch lengths and the exponential weighted average - epoch_lengths, exponential_weighted_average_epoch_length = get_exponential_weighted_epoch_lengths( - args.url, next_epoch_id, args.num_past_epochs, args.decay_rate) + epoch_lengths, exponential_weighted_average_epoch_length = ( + get_exponential_weighted_epoch_lengths( + args.url, next_epoch_id, args.num_past_epochs, args.decay_rate + ) + ) # Predict future epoch start dates future_epochs = predict_future_epochs( - current_timestamp, exponential_weighted_average_epoch_length, - args.num_future_epochs, args.timezone) + current_timestamp, + exponential_weighted_average_epoch_length, + args.num_future_epochs, + args.timezone, + ) if args.voting_date: - find_protocol_upgrade_time(args.voting_date, future_epochs, - args.timezone) + find_protocol_upgrade_time(args.voting_date, future_epochs, args.timezone) elif args.voting_date_day: find_best_voting_hour(args.voting_date_day, future_epochs) @@ -234,54 +238,61 @@ def main(args): class SetURLFromChainID(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): - if values == 'mainnet': - setattr(namespace, 'url', 'https://archival-rpc.mainnet.near.org') - elif values == 'testnet': - setattr(namespace, 'url', 'https://archival-rpc.testnet.near.org') + if values == "mainnet": + setattr(namespace, "url", "https://archival-rpc.mainnet.near.org") + elif values == "testnet": + setattr(namespace, "url", "https://archival-rpc.testnet.near.org") # Set up command-line argument parsing if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Approximate future epoch start dates for NEAR Protocol.") + description="Approximate future epoch start dates for NEAR Protocol." + ) # Create a mutually exclusive group for chain_id and url group = parser.add_mutually_exclusive_group(required=False) group.add_argument("--url", help="The RPC URL to query.") group.add_argument( "--chain_id", - choices=['mainnet', 'testnet'], + choices=["mainnet", "testnet"], action=SetURLFromChainID, - help= - "The chain ID (either 'mainnet' or 'testnet'). Sets the corresponding URL." + help="The chain ID (either 'mainnet' or 'testnet'). Sets the corresponding URL.", ) - parser.add_argument("--num_past_epochs", - type=int, - default=4, - help="Number of past epochs to analyze.") - parser.add_argument("--decay_rate", - type=float, - default=0.1, - help="Decay rate for exponential weighting.") - parser.add_argument("--num_future_epochs", - type=int, - default=10, - help="Number of future epochs to predict.") + parser.add_argument( + "--num_past_epochs", + type=int, + default=4, + help="Number of past epochs to analyze.", + ) + parser.add_argument( + "--decay_rate", + type=float, + default=0.1, + help="Decay rate for exponential weighting.", + ) + parser.add_argument( + "--num_future_epochs", + type=int, + default=10, + help="Number of future epochs to predict.", + ) parser.add_argument( "--timezone", type=valid_timezone, default="UTC", - help="Time zone to display times in (e.g., 'America/New_York').") + help="Time zone to display times in (e.g., 'America/New_York').", + ) # Voting date arguments voting_group = parser.add_mutually_exclusive_group() voting_group.add_argument( "--voting_date", type=valid_voting_datetime, - help="Voting date in 'YYYY-MM-DD HH:MM:SS' format.") + help="Voting date in 'YYYY-MM-DD HH:MM:SS' format.", + ) voting_group.add_argument( "--voting_date_day", - help= - "Voting date (day) in 'YYYY-MM-DD' format to find voting hours resulting in upgrade during working hours." + help="Voting date (day) in 'YYYY-MM-DD' format to find voting hours resulting in upgrade during working hours.", ) args = parser.parse_args() diff --git a/debug_scripts/request_chain_info.py b/debug_scripts/request_chain_info.py index d5fe22b5665..f51a8b27022 100755 --- a/debug_scripts/request_chain_info.py +++ b/debug_scripts/request_chain_info.py @@ -4,64 +4,64 @@ import json import argparse -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description='This is a script to request for blockchain info') - parser.add_argument('--chain', - choices=['mainnet', 'testnet', 'betanet'], - required=True) - parser.add_argument('--archive', - action='store_true', - help='whether to request from archival nodes') - parser.add_argument('--method', - choices=['block', 'chunk'], - required=True, - help='type of request') - parser.add_argument('--block_id', - type=str, - help='block id, can be either block height or hash') - parser.add_argument('--shard_id', type=int, help='shard id for the chunk') - parser.add_argument('--chunk_id', type=str, help='chunk hash') - parser.add_argument('--result_key', - type=str, - nargs='*', - help='filter results by these keys') + description="This is a script to request for blockchain info" + ) + parser.add_argument( + "--chain", choices=["mainnet", "testnet", "betanet"], required=True + ) + parser.add_argument( + "--archive", action="store_true", help="whether to request from archival nodes" + ) + parser.add_argument( + "--method", choices=["block", "chunk"], required=True, help="type of request" + ) + parser.add_argument( + "--block_id", type=str, help="block id, can be either block height or hash" + ) + parser.add_argument("--shard_id", type=int, help="shard id for the chunk") + parser.add_argument("--chunk_id", type=str, help="chunk hash") + parser.add_argument( + "--result_key", type=str, nargs="*", help="filter results by these keys" + ) args = parser.parse_args() - url = 'https://{}.{}.near.org'.format( - 'archival-rpc' if args.archive else 'rpc', args.chain) + url = "https://{}.{}.near.org".format( + "archival-rpc" if args.archive else "rpc", args.chain + ) def get_block_id(block_id): if block_id.isnumeric(): return int(block_id) return block_id - if args.method == 'block': + if args.method == "block": if args.block_id is not None: - params = {'block_id': get_block_id(args.block_id)} + params = {"block_id": get_block_id(args.block_id)} else: - params = {'finality': 'final'} - elif args.method == 'chunk': + params = {"finality": "final"} + elif args.method == "chunk": if args.shard_id is not None: assert args.block_id is not None params = { - 'shard_id': args.shard_id, - 'block_id': get_block_id(args.block_id) + "shard_id": args.shard_id, + "block_id": get_block_id(args.block_id), } elif args.chunk_id is not None: - params = {'chunk_id': args.chunk_id} + params = {"chunk_id": args.chunk_id} else: assert False payload = { - 'jsonrpc': '2.0', - 'id': 'dontcare', - 'method': args.method, - 'params': params + "jsonrpc": "2.0", + "id": "dontcare", + "method": args.method, + "params": params, } response = requests.post(url, json=payload) - result = response.json()['result'] + result = response.json()["result"] if args.result_key is not None: for key in args.result_key: result = result[key] diff --git a/debug_scripts/send_validator_logs.py b/debug_scripts/send_validator_logs.py index 51c2c92c601..65890f4e2da 100644 --- a/debug_scripts/send_validator_logs.py +++ b/debug_scripts/send_validator_logs.py @@ -8,8 +8,9 @@ import urllib.parse -def filter_log_file(log_file: str, start_time: datetime.datetime, - end_time: datetime.datetime) -> list: +def filter_log_file( + log_file: str, start_time: datetime.datetime, end_time: datetime.datetime +) -> list: """ Filter log file for a time range. start_time: datetime.datetime @@ -28,9 +29,9 @@ def filter_log_file(log_file: str, start_time: datetime.datetime, for line in f: # [0m and [2m are ANSI shell color codes. Removing them to parse dates. split_lines = line.split("[0m", 1)[0].replace("\x1b[2m", "") - dt = datetime.datetime.strptime( - split_lines[:-5], - "%b %d %H:%M:%S").replace(year=datetime.datetime.now().year) + dt = datetime.datetime.strptime(split_lines[:-5], "%b %d %H:%M:%S").replace( + year=datetime.datetime.now().year + ) if start_time <= dt <= end_time: filtered_logs.append(line) return filtered_logs @@ -50,43 +51,44 @@ def upload_to_s3(file_lines: list, account: str) -> str: for line in file_lines: file_string.write(line) + # cspell:words getsizeof file_obj = io.BytesIO(file_string.getvalue().encode()) gzipped_content = gzip.compress(file_obj.read()) print( f"uploading compressed file. File size is: {sys.getsizeof(gzipped_content)} Bytes" ) - s3 = boto3.resource('s3') - s3.Bucket(BUCKET).upload_fileobj(io.BytesIO(gzipped_content), - f"logs/{s3_destination}") - s3_link = f"https://{BUCKET}.s3.amazonaws.com/logs/{urllib.parse.quote(s3_destination)}" + s3 = boto3.resource("s3") + s3.Bucket(BUCKET).upload_fileobj( + io.BytesIO(gzipped_content), f"logs/{s3_destination}" + ) + s3_link = ( + f"https://{BUCKET}.s3.amazonaws.com/logs/{urllib.parse.quote(s3_destination)}" + ) print(f"Log File was uploaded to S3: {s3_link}") file_obj.close() return s3_link -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Send logs to near.') - parser.add_argument('--log_file', - type=str, - help='Absolute path to log file.', - required=True) - parser.add_argument('--account', - type=str, - help='Near account id.', - required=True) - parser.add_argument('--last_seconds', - type=int, - help='Filter logs for last x seconds.', - required=True) +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Send logs to near.") + parser.add_argument( + "--log_file", type=str, help="Absolute path to log file.", required=True + ) + parser.add_argument("--account", type=str, help="Near account id.", required=True) + parser.add_argument( + "--last_seconds", + type=int, + help="Filter logs for last x seconds.", + required=True, + ) args = parser.parse_args() log_file_path = args.log_file end_timestamp = datetime.datetime.utcnow() - start_timestamp = end_timestamp - datetime.timedelta( - seconds=args.last_seconds) + start_timestamp = end_timestamp - datetime.timedelta(seconds=args.last_seconds) - filtered_log_lines = filter_log_file(log_file=args.log_file, - start_time=start_timestamp, - end_time=end_timestamp) + filtered_log_lines = filter_log_file( + log_file=args.log_file, start_time=start_timestamp, end_time=end_timestamp + ) upload_to_s3(file_lines=filtered_log_lines, account=args.account) diff --git a/debug_scripts/tests/send_validator_logs_test.py b/debug_scripts/tests/send_validator_logs_test.py index ce7679d39fd..07dbd4c751e 100644 --- a/debug_scripts/tests/send_validator_logs_test.py +++ b/debug_scripts/tests/send_validator_logs_test.py @@ -10,15 +10,14 @@ class test_validator_log_filtering(unittest.TestCase): def test_time_filtering(self): start_time = datetime.datetime(2022, 4, 4, 23, 42, 0, 0) end_time = datetime.datetime(2022, 4, 4, 23, 49, 0, 0) - output_file_obj = filter_log_file("./tests/data/node0.logs", - start_time=start_time, - end_time=end_time) + output_file_obj = filter_log_file( + "./tests/data/node0.logs", start_time=start_time, end_time=end_time + ) self.assertIsInstance( - output_file_obj, list, - "Parsed file object should be of type io.BytesIO") - self.assertEqual(len(output_file_obj), 48, - "Filtered log should have 48 lines") + output_file_obj, list, "Parsed file object should be of type io.BytesIO" + ) + self.assertEqual(len(output_file_obj), 48, "Filtered log should have 48 lines") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/docs/advanced_configuration/networking.md b/docs/advanced_configuration/networking.md index 9914d2cd0bc..a29ac089039 100644 --- a/docs/advanced_configuration/networking.md +++ b/docs/advanced_configuration/networking.md @@ -1,7 +1,6 @@ This document describes the advanced network options that you can configure by modifying the "network" section of your "config.json" file: - ``` { // ... diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 71803744707..f5f321662cd 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -263,7 +263,7 @@ more information on the usage. Rust has built-in support for writing unit tests by marking functions with the `#[test]` directive. Take full advantage of that! Testing not only confirms that what was written works the way it was intended to but -also helps during refactoring since it catches unintended behaviour +also helps during refactoring since it catches unintended behavior changes. Not all tests are created equal though and while some may only need diff --git a/docs/architecture/gas/estimator.md b/docs/architecture/gas/estimator.md index a243aa96069..ab01300f5d2 100644 --- a/docs/architecture/gas/estimator.md +++ b/docs/architecture/gas/estimator.md @@ -66,6 +66,7 @@ But when you run estimations and especially when you want to interpret the results, you want to understand the metric used. Available metrics are `time` and `icount`. + Starting with `time`, this is a simple wall-clock time measurement. At the end of the day, this is what counts in a validator setup. But unfortunately, this metric is very dependent on the specific hardware and what else is running on diff --git a/docs/architecture/gas/gas_profile.md b/docs/architecture/gas/gas_profile.md index 2a73b30fd6c..0cf9bd7e113 100644 --- a/docs/architecture/gas/gas_profile.md +++ b/docs/architecture/gas/gas_profile.md @@ -21,6 +21,7 @@ NEAR_ENV=mainnet near tx-status 8vYxsqYp5Kkfe8j9LsTqZRsEupNkAs1WvgcGcUE4MUUw \ --nodeUrl https://archival-rpc.mainnet.near.org # Allows to retrieve older transactions. ``` + ``` Transaction app.nearcrowd.near:8vYxsqYp5Kkfe8j9LsTqZRsEupNkAs1WvgcGcUE4MUUw { diff --git a/docs/architecture/how/README.md b/docs/architecture/how/README.md index 0e2586625e0..36bd8b29714 100644 --- a/docs/architecture/how/README.md +++ b/docs/architecture/how/README.md @@ -42,8 +42,8 @@ There are several important actors in neard: to **client**. It only accesses data stored in a node’s storage and does not mutate any state. It is used for two purposes: - * Answering RPC requests by fetching the relevant piece of data from storage. - * Handling some network requests that do not require any changes to the + * Answering RPC requests by fetching the relevant piece of data from storage. + * Handling some network requests that do not require any changes to the storage, such as header sync, state sync, and block sync requests. `ViewClientActor` runs in four threads by default but this number is configurable. @@ -54,12 +54,10 @@ Flow for incoming messages: ![](https://user-images.githubusercontent.com/1711539/195619986-25798cde-8a91-4721-86bd-93fa924b483a.png) - Flow for outgoing messages: ![](https://user-images.githubusercontent.com/1711539/195626792-7697129b-7f9c-4953-b939-0b9bcacaf72c.png) - ## How neard operates when it is fully synced When a node is fully synced, the main logic of the node operates in the @@ -106,7 +104,6 @@ The main logic is illustrated below: ![](https://user-images.githubusercontent.com/1711539/195635652-f0c7ebae-a2e5-423f-8e62-b853b815fcec.png) - ## How neard works when it is synchronizing `PeerManagerActor` periodically sends a `NetworkInfo` message to `ClientActor` @@ -144,14 +141,14 @@ to sync. The synchronization process is done in three steps: that contains some metadata the node needs to know about. Then the node computes the number of state parts it needs to download and requests those parts from different peers who track the shard. - 4. After all parts are downloaded, the node [combines those state + 4. After all parts are downloaded, the node [combines those state parts](https://github.com/near/nearcore/blob/279044f09a7e6e5e3f26db4898af3655dae6eda6/chain/client/src/client_actor.rs#L1877) and then [finalizes](https://github.com/near/nearcore/blob/279044f09a7e6e5e3f26db4898af3655dae6eda6/chain/chain/src/chain.rs#L3065) the state sync by applying the last chunk included in or before the sync block so that the node has the state after applying sync block to be able to apply the next block. - 5. The node [resets + 5. The node [resets heads](https://github.com/near/nearcore/blob/279044f09a7e6e5e3f26db4898af3655dae6eda6/chain/chain/src/chain.rs#L1874) properly after state sync. @@ -173,28 +170,28 @@ head height, the block is simply dropped. ClientActor has some periodically running routines that are worth noting: * [Doomslug - timer](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1198) - - This routine runs every `doosmslug_step_period` (set to 100ms by default) and + timer](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1198) - + This routine runs every `doomslug_step_period` (set to 100ms by default) and updates consensus information. If the node is a validator node, it also sends approvals when necessary. * [Block - production](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L991) - + production](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L991) - This routine runs every `block_production_tracking_delay` (which is set to 100ms by default) and checks if the node should produce a block. * [Log - summary](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1790) - + summary](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1790) - Prints a log line that summarizes block rate, average gas used, the height of the node, etc. every 10 seconds. * [Resend chunk - requests](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/chunks/src/lib.rs#L910) - + requests](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/chunks/src/lib.rs#L910) - This routine runs every `chunk_request_retry_period` (which is set to 400ms). - It resends the chunk part requests for those that are not yet responded to. -* [Sync](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1629) - + It resend the chunk part requests for those that are not yet responded to. +* [Sync](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1629) - This routine runs every `sync_step_period` (which is set to 10ms by default) and checks whether the node needs to sync from its peers and, if needed, also starts the syncing process. * [Catch - up](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1581) - + up](https://github.com/near/nearcore/blob/fa78002a1b4119e5efe277c3073b3f333f451ffc/chain/client/src/client_actor.rs#L1581) - This routine runs every `catchup_step_period` (which is set to 100ms by default) and runs the catch up process. This only applies if a node validates shard A in epoch X and is going to validate a different shard B in epoch X+1. diff --git a/docs/architecture/how/cross-shard.md b/docs/architecture/how/cross-shard.md index 168ec882579..cc25bfeb6e0 100644 --- a/docs/architecture/how/cross-shard.md +++ b/docs/architecture/how/cross-shard.md @@ -9,7 +9,7 @@ shard 1). Imagine, we run the following command in the command line: ```console -$ NEAR_ENV=local near send shard0 shard1 500 +NEAR_ENV=local near send shard0 shard1 500 ``` What happens under the hood? How is this transaction changed into receipts and @@ -125,7 +125,7 @@ Chunk: Ok( **Side note:** When we're converting the transaction into a receipt, we also use this moment to deduct prepaid gas fees and transferred tokens from the 'signer' -account. The details on how much gas is charged can be found at https://nomicon.io/RuntimeSpec/Fees/. +account. The details on how much gas is charged can be found at . ## Step 2 - cross shard receipt @@ -209,7 +209,7 @@ Such a receipt is sent to the destination shard (we'll explain this process in a separate article) where it can be executed. -## 3. Gas refund. +## 3. Gas refund When shard 1 processes the receipt above, it is then ready to refund the unused gas to the original account (`shard0`). So it also creates the receipt, and puts diff --git a/docs/architecture/how/gas.md b/docs/architecture/how/gas.md index 2de91364596..275ba055596 100644 --- a/docs/architecture/how/gas.md +++ b/docs/architecture/how/gas.md @@ -17,7 +17,7 @@ The topic is split into several sections. - [Burning Gas](#burning-gas): Who receives burnt tokens? - [Gas in Contract Calls](#gas-in-contract-calls): How is gas attached to calls? - [Contract Reward](#contract-reward): How smart contract earn a reward. -2. [Gas Price](#gas-price): +2. [Gas Price](#gas-price): - [Block-Level Gas Price](#block-level-gas-price): How the block-level gas price is determined. - [Pessimistic Gas Price](#pessimistic-gas-price): How worst-case gas pricing is estimated. - [Effective Gas Purchase Cost](#effective-gas-purchase-cost): The cost paid for a receipt. @@ -84,10 +84,10 @@ draws gas from the `attached_gas`, sometimes also called `prepaid_gas`, until it reaches zero, at which point the function call aborts with a `GasExceeded` error. No changes are persisted on chain. -(*Note on naming: If you see `prepaid_fee: Balance` in the nearcore code base, +(_Note on naming: If you see `prepaid_fee: Balance` in the nearcore code base, this is NOT only the fee for `prepaid_gas`. It also includes prepaid fees for other gas costs. However, `prepaid_gas: Gas` is used the same in the code base -as described in this document.*) +as described in this document._) Attaching gas to function calls is the primary way for end-users and contract developers to interact with gas. All other gas fees are implicitly computed and @@ -128,7 +128,6 @@ gas that is computed after the current call has finished. This allows attaching attaching an equal fraction to each, or any other split as defined by the weight per call. - ### Contract Reward A rather unique property of Near Protocol is that a part of the gas fee goes to @@ -144,7 +143,7 @@ downsides, such as when implementing a free meta-transaction relayer one has to be careful not to be susceptible to faucet-draining attacks where an attacker extracts funds from the relayer by making calls to a contract they own. -How much contracts receive from execution depends on two things. +How much contracts receive from execution depends on two things. 1. How much gas is burnt on the function call execution itself. That is, only the gas taken from the `attached_gas` of a function call is considered for @@ -164,7 +163,6 @@ For brevity, `gas_burnt_for_function_call` in the diagram is denoted as `wasm fe ![Slightly Simplified Gas Flow Diagram](https://github.com/near/nearcore/assets/6342444/32600ef0-1475-43af-b196-576317787578) - ## Gas Price Gas pricing is a surprisingly deep and complicated topic. Usually, we only think @@ -202,7 +200,6 @@ compute the total capacity as the sum of gas limits stored in the chunk headers to be compliant. Using a hard-coded `1000 Tgas * num_shards` would lead to incorrect block header validation. - ### Pessimistic Gas Price The pessimistic gas price calculation uses the fact that any transaction can @@ -224,8 +221,7 @@ pessimistic(current_gas_price, max_depth) = current_gas_price × 1.03^max_depth This still is not the price at which gas is purchased. But we are very close. - -### Effective Gas Purchase Cost +### Effective Gas Purchase Cost When a transaction is converted to its root action receipt, the gas costs are calculated in two parts. @@ -253,7 +249,6 @@ resulting higher number. Diagram](https://github.com/near/nearcore/assets/6342444/8341fb45-9beb-4808-8a89-8144fa075930) - ## Tracking Gas in Receipts The previous section explained how gas is bought and what determines its price. diff --git a/docs/architecture/how/gc.md b/docs/architecture/how/gc.md index 9d22a975f28..9fccf9c66e9 100644 --- a/docs/architecture/how/gc.md +++ b/docs/architecture/how/gc.md @@ -12,7 +12,7 @@ at most 2 blocks from the chain. For more details look at function `clear_data()` in file `chain/chain/src/chain.rs` -## How it works: +## How it works Imagine the following chain (with 2 forks) diff --git a/docs/architecture/how/meta-tx.md b/docs/architecture/how/meta-tx.md index d19c6f196b4..e7a851ee36b 100644 --- a/docs/architecture/how/meta-tx.md +++ b/docs/architecture/how/meta-tx.md @@ -13,12 +13,12 @@ more flexible. ## Overview + ![Flow chart of meta transactions](https://raw.githubusercontent.com/near/NEPs/003e589e6aba24fc70dd91c9cf7ef0007ca50735/neps/assets/nep-0366/NEP-DelegateAction.png) _Credits for the diagram go to the NEP authors Alexander Fadeev and Egor Uleyskiy._ - The graphic shows an example use case for meta transactions. Alice owns an amount of the fungible token $FT. She wants to transfer some to John. To do that, she needs to call `ft_transfer("john", 10)` on an account named `FT`. @@ -67,7 +67,7 @@ In the example visualized above, the payment is done using $FT. Together with the transfer to John, Alice also adds an action to pay 0.1 $FT to the relayer. The relayer checks the content of the `SignedDelegateAction` and only processes it if this payment is included as the first action. In this way, the relayer -will be paid in the same transaction as John. +will be paid in the same transaction as John. Note that the payment to the relayer is still not guaranteed. It could be that Alice does not have sufficient $FT and the transfer fails. To mitigate, the @@ -124,12 +124,12 @@ having multiple receivers in a delegate action. Naturally, it runs into all the same complications (false sense of atomicity) and ends with the same conclusion: Omitted from the MVP and left open for future improvement. -## Limitation: Accounts must be initialized +## Limitation: Accounts must be initialized Any transaction, including meta transactions, must use NONCEs to avoid replay attacks. The NONCE must be chosen by Alice and compared to a NONCE stored on chain. This NONCE is stored on the access key information that gets initialized -when creating an account. +when creating an account. Implicit accounts don't need to be initialized in order to receive NEAR tokens, or even $FT. This means users could own $FT but no NONCE is stored on chain for diff --git a/docs/architecture/how/proofs.md b/docs/architecture/how/proofs.md index 8d645368853..32abf219c57 100644 --- a/docs/architecture/how/proofs.md +++ b/docs/architecture/how/proofs.md @@ -19,7 +19,7 @@ But there is actually a better solution - that doesn’t require you to trust th single (or many) RPC nodes, and to verify, by yourself, that your transaction was actually executed. -## Let’s talk about proofs (merkelization): +## Let’s talk about proofs (merkelization) Imagine you have 4 values that you’d like to store, in such a way, that you can easily prove that a given value is present. @@ -133,7 +133,6 @@ tree the sibling is), on the path to the root. **Note:** proof section doesn’t contain the root itself and also doesn’t include the hash of the receipt. - ## [Advanced section]: Let’s look at a concrete example Imagine that we have the following receipt: @@ -182,7 +181,7 @@ I know that the receipt should belong to Shard 3 so let the chunk header: ```console -$ neard view-state chunks --chunk-hash 5ub8CZMQmzmZYQcJU76hDC3BsajJfryjyShxGF9rzpck +neard view-state chunks --chunk-hash 5ub8CZMQmzmZYQcJU76hDC3BsajJfryjyShxGF9rzpck ``` ``` diff --git a/docs/architecture/how/receipt-congestion.md b/docs/architecture/how/receipt-congestion.md index 34f51eec3eb..b7aab76d5a9 100644 --- a/docs/architecture/how/receipt-congestion.md +++ b/docs/architecture/how/receipt-congestion.md @@ -31,8 +31,8 @@ Okay, we have the capacities of the network modeled. Now let's look at how a receipt execution maps onto it. Let's say a receipt starts at shard 1 with 300 Tgas. While executing, it burns 100 Tgas and -creates an outgoing receipts with 200 Tgas to another shard. We can represent this in the flow network with -100 Tgas to the sink of shard 1 and 200 Tgas to shard 2. +creates an outgoing receipts with 200 Tgas to another shard. We can represent this in the flow network with +100 Tgas to the sink of shard 1 and 200 Tgas to shard 2. ![graph](../../images/congestion/receipt_flow_example_0.svg) @@ -89,7 +89,6 @@ cases. We are instead trying to find a solution that can give 100% utilization for normal operation and then falls back to `1000 Tgas / NUM_SHARDS` when it has to, in order to prevent out-of-memory crashes. - ## Idea 2: Limit transactions when we use too much memory What if we have no limit at the source until we notice we are above the memory @@ -157,7 +156,7 @@ bytes, because it is here to maximize utilization, not to minimize memory consumption. And utilization is best measured in gas. If we have a queue of 10_000 Tgas waiting, even if only 10% of that is burnt in this step of the transaction, we still have 1000 Tgas of useful work we can contribute to the -total flow. Thus under the assumption that at least 10% of gas is being burnt, +total flow. Thus under the assumption that at least 10% of gas is being burnt, we have 100% utilization. A limit in bytes would be better to argue how much memory we need exactly. But @@ -182,7 +181,6 @@ eventually end up with full buffers. Combined with idea 2, eventually all transactions to those shards are rejected. All of this without affecting shards that are not on the critical path. - ## Putting it all together The proposal in [NEP-539](https://github.com/near/NEPs/blob/master/neps/nep-0539.md) combines all diff --git a/docs/architecture/how/resharding_v2.md b/docs/architecture/how/resharding_v2.md index 46c959646be..a7b5d3dc600 100644 --- a/docs/architecture/how/resharding_v2.md +++ b/docs/architecture/how/resharding_v2.md @@ -6,31 +6,33 @@ Resharding is the process in which the shard layout changes. The primary purpose of resharding is to keep the shards small so that a node meeting minimum hardware -requirements can safely keep up with the network while tracking some set minimum -number of shards. +requirements can safely keep up with the network while tracking some set minimum +number of shards. ## Specification The resharding is described in more detail in the following NEPs: + * [NEP-0040](https://github.com/near/NEPs/blob/master/specs/Proposals/0040-split-states.md) * [NEP-0508](https://github.com/near/NEPs/blob/master/neps/nep-0508.md) ## Shard layout The shard layout determines the number of shards and the assignment of accounts - to shards (as single account cannot be split between shards). + to shards (as single account cannot be split between shards). + +There are two versions of the ShardLayout enum. -There are two versions of the ShardLayout enum. * v0 - maps the account to a shard taking hash of the account id modulo number of shards * v1 - maps the account to a shard by looking at a set of predefined boundary accounts and selecting the shard where the accounts fits by using alphabetical order At the time of writing there are three pre-defined shard layouts but more can -be added in the future. +be added in the future. -* v0 - The first shard layout that contains only a single shard encompassing all the accounts. -* simple nightshade - Splits the accounts into 4 shards. -* simple nightshade v2 - Splits the accounts into 5 shards. +* v0 - The first shard layout that contains only a single shard encompassing all the accounts. +* simple nightshade - Splits the accounts into 4 shards. +* simple nightshade v2 - Splits the accounts into 5 shards. **IMPORTANT**: Using alphabetical order applies to the full account name, so ``a.near`` could belong to shard 0, while ``z.a.near`` to shard 3. @@ -43,21 +45,19 @@ In the near future we are planning on switching to simple nightshade v2 (which i ``vec!["aurora", "aurora-0", "kkuuue2akv_1630967379.near", "tge-lockup.sweat"]`` - ## Shard layout changes -Shard Layout is determined at epoch level in the AllEpochConfig based on the protocol version of the epoch. +Shard Layout is determined at epoch level in the AllEpochConfig based on the protocol version of the epoch. -The shard layout can change at the epoch boundary. Currently in order to change the +The shard layout can change at the epoch boundary. Currently in order to change the shard layout it is necessary to manually determine the new shard layout and setting it for the desired protocol version in the ``AllEpochConfig``. - ### Deeper technical details It all starts in ``preprocess_block`` - if the node sees, that the block it is about to preprocess is the first block of the epoch (X+1) - it calls -``get_state_sync_info``, which is responsible for figuring out which shards will +``get_state_sync_info``, which is responsible for figuring out which shards will be needed in next epoch (X+2). This is the moment, when node can request new shards that it didn't track before (using StateSync) - and if it detects that the shard layout would change in the next epoch, it also involves the StateSync - but skips the download part (as it already has the data) - and starts from resharding. @@ -66,7 +66,7 @@ StateSync in this phase would send the ``ReshardingRequest`` to the ``SyncJobsAc We'd use the background thread to perform resharding: the goal is to change the one trie (that represents the state of the current shard) - to multiple tries (one for each of the new shards). -In order to split a trie into children tries we use a snapshot of the flat storage. We iterate over all of the entries in the flat storage and we build the children tries by inserting the parent entry into either of the children tries. +In order to split a trie into children tries we use a snapshot of the flat storage. We iterate over all of the entries in the flat storage and we build the children tries by inserting the parent entry into either of the children tries. Extracting of the account from the key happens in ``parse_account_id_from_raw_key`` - and we do it for all types of data that we store in the trie (contract code, keys, account info etc) EXCEPT for Delayed receipts. Then, we figure out the shard that this account is going to belong to, and we add this key/value to that new trie. @@ -74,18 +74,17 @@ This way, after going over all the key/values from the original trie, we end up IMPORTANT: in the current code, we only support such 'splitting' (so a new shard can have just one parent). - ### Why delayed receipts are special? + For all the other columns, there is no dependency between entries, but in case of delayed receipts - we are forming a 'queue'. We store the information about the first index and the last index (in DelayedReceiptIndices struct). Then, when receipt arrives, we add it as the 'DELAYED_RECEIPT + last_index' key (and increment last_index by 1). -That is why we cannot move this trie entry type in the same way as others where account id is part of the key. Instead we do it by iterating over this queue and inserting entries to the queue of the relevant child shard. - +That is why we cannot move this trie entry type in the same way as others where account id is part of the key. Instead we do it by iterating over this queue and inserting entries to the queue of the relevant child shard. ## Constraints -The state sync of the parent shard, the resharing and the catchup of the children shards must all complete within a single epoch. +The state sync of the parent shard, the resharding and the catchup of the children shards must all complete within a single epoch. ## Rollout @@ -93,29 +92,28 @@ The state sync of the parent shard, the resharing and the catchup of the childre The resharding will be initiated by having it included in a dedicated protocol version together with neard. Here is the expected flow of events: -* A new neard release is published and protocol version upgrade date is set to D, roughly a week from the release. -* All node operators upgrade their binaries to the newly released version within the given timeframe, ideally as soon as possible but no later than D. -* The protocol version upgrade voting takes place at D in an epoch E and nodes vote in favour of switching to the new protocol version in epoch E+2. -* The resharding begins at the beginning of epoch E+1. -* The network switches to the new shard layout in the first block of epoch E+2. - +* A new neard release is published and protocol version upgrade date is set to D, roughly a week from the release. +* All node operators upgrade their binaries to the newly released version within the given time frame, ideally as soon as possible but no later than D. +* The protocol version upgrade voting takes place at D in an epoch E and nodes vote in favour of switching to the new protocol version in epoch E+2. +* The resharding begins at the beginning of epoch E+1. +* The network switches to the new shard layout in the first block of epoch E+2. ### Monitoring -Resharding exposes a number of metrics and logs that allow for monitoring the resharding process as it is happening. Resharding requires manual recovery in case anything goes wrong and should be monitored in order to ensure smooth node operation. +Resharding exposes a number of metrics and logs that allow for monitoring the resharding process as it is happening. Resharding requires manual recovery in case anything goes wrong and should be monitored in order to ensure smooth node operation. * near_resharding_status is the primary metric that should be used for tracking the progress of resharding. It's tagged with a shard_uid label of the parent shard. It's set to corresponding ReshardingStatus enum and can take one of the following values - * 0 - Scheduled - resharding is scheduled and waiting to be executed. - * 1 - Building - resharding is running. Only one shard at a time can be in that state while the rest will be either finished or waiting in the Scheduled state. - * 2 - Finished - resharding is finished. - * -1 - Failed - resharding failed and manual recovery action is required. The node will operate as usual until the end of the epoch but will then stop being able to process blocks. -* near_resharding_batch_size and near_resharding_batch_count - those two metrics show how much data has been resharded. Both metrics should progress with the near_resharding_status as follows. - * While in the Scheduled state both metrics should remain 0. - * While in the Building state both metrics should be gradually increasing. - * While in the Finished state both metrics should remain at the same value. -* near_resharding_batch_prepare_time_bucket, near_resharding_batch_apply_time_bucket and near_resharding_batch_commit_time_bucket - those three metrics can be used to track the performance of resharding and fine tune throttling if needed. As a rule of thumb the combined time of prepare, apply and commit for a batch should remain at the 100ms-200ms level on average. Higher batch processing time may lead to disruptions in block processing, missing chunks and blocks. - -Here are some example metric values when finished for different shards and networks. The duration column reflects the duration of the building phase. Those were captured in production like environment in November 2023 and actual times at the time of resharding in production may be slightly higher. + * 0 - Scheduled - resharding is scheduled and waiting to be executed. + * 1 - Building - resharding is running. Only one shard at a time can be in that state while the rest will be either finished or waiting in the Scheduled state. + * 2 - Finished - resharding is finished. + * -1 - Failed - resharding failed and manual recovery action is required. The node will operate as usual until the end of the epoch but will then stop being able to process blocks. +* near_resharding_batch_size and near_resharding_batch_count - those two metrics show how much data has been resharded. Both metrics should progress with the near_resharding_status as follows. + * While in the Scheduled state both metrics should remain 0. + * While in the Building state both metrics should be gradually increasing. + * While in the Finished state both metrics should remain at the same value. +* near_resharding_batch_prepare_time_bucket, near_resharding_batch_apply_time_bucket and near_resharding_batch_commit_time_bucket - those three metrics can be used to track the performance of resharding and fine tune throttling if needed. As a rule of thumb the combined time of prepare, apply and commit for a batch should remain at the 100ms-200ms level on average. Higher batch processing time may lead to disruptions in block processing, missing chunks and blocks. + +Here are some example metric values when finished for different shards and networks. The duration column reflects the duration of the building phase. Those were captured in production like environment in November 2023 and actual times at the time of resharding in production may be slightly higher. | mainnet | duration | batch count | batch size | | ------- | -------- | ----------- | ---------- | @@ -133,46 +131,41 @@ Here are some example metric values when finished for different shards and netwo | shard 2 | 2h31min | 75,529 | 75.6GB | | shard 3 | 2h22min | 63,621 | 49.2GB | - Here is an example of what that may look like in a grafana dashboard. Please keep in mind that the values and duration is not representative as the sample data below is captured in a testing environment with different configuration. - Screenshot 2023-12-01 at 10 10 20 Screenshot 2023-12-01 at 10 10 50 Screenshot 2023-12-01 at 10 10 42 - - ### Throttling -The resharding process can be quite resource intensive and affect the regular operation of a node. In order to mitigate that as well as limit any need for increasing hardware specifications of the nodes throttling was added. Throttling slows down resharding to not have it impact other node operations. Throttling can be configured by adjusting the resharding_config in the node config file. +The resharding process can be quite resource intensive and affect the regular operation of a node. In order to mitigate that as well as limit any need for increasing hardware specifications of the nodes throttling was added. Throttling slows down resharding to not have it impact other node operations. Throttling can be configured by adjusting the resharding_config in the node config file. * batch_size - controls the size of batches in which resharding moves data around. Setting a smaller batch size will slow down the resharding process and make it less resource-consuming. -* batch_delay - controls the delay between processing of batches. Setting a smaller batch delay will speed up the resharding process and make it more resource-consuming. +* batch_delay - controls the delay between processing of batches. Setting a smaller batch delay will speed up the resharding process and make it more resource-consuming. -The remaining fields in the ReshardingConfig are only intended for testing purposes and should remain set to their default values. +The remaining fields in the ReshardingConfig are only intended for testing purposes and should remain set to their default values. -The default configuration for ReshardingConfig should provide a good and safe setting for resharding in the production networks. There is no need for node operators to make any changes to it unless they observe issues. +The default configuration for ReshardingConfig should provide a good and safe setting for resharding in the production networks. There is no need for node operators to make any changes to it unless they observe issues. -The resharding config can be adjusted at runtime, without restarting the node. The config needs to be updated first and then a SIGHUP signal should be sent to the neard process. When received the signal neard will update the config and print a log message showing what fields were changed. It's recommended to check the log to make sure the relevant config change was correctly picked up. +The resharding config can be adjusted at runtime, without restarting the node. The config needs to be updated first and then a SIGHUP signal should be sent to the neard process. When received the signal neard will update the config and print a log message showing what fields were changed. It's recommended to check the log to make sure the relevant config change was correctly picked up. ## Future possibilities ### Localize resharding to a single shard -Currently when resharding we need to move the data for all shards even if only a single shard is being split. That is due to having the version field in the storage key that needs to be updated when changing shard layout version. +Currently when resharding we need to move the data for all shards even if only a single shard is being split. That is due to having the version field in the storage key that needs to be updated when changing shard layout version. -This can be improved by changing how ShardUId works e.g. removing the version and instead using globally unique shard ids. +This can be improved by changing how ShardUId works e.g. removing the version and instead using globally unique shard ids. ### Dynamic resharding -The current implementation relies on having the shard layout determined offline and manually added to the node implementation. +The current implementation relies on having the shard layout determined offline and manually added to the node implementation. The dynamic resharding would mean that the network itself can automatically determine that resharding is needed, what should be the new shard layout and schedule the resharding. - ### Support different changes to shard layout -The current implementation only supports splitting a shard. In the future we can consider adding support for other operations such as merging two shards or moving an existing boundary account. +The current implementation only supports splitting a shard. In the future we can consider adding support for other operations such as merging two shards or moving an existing boundary account. diff --git a/docs/architecture/how/serialization.md b/docs/architecture/how/serialization.md index 9f4455c62fb..5c60c2aec0c 100644 --- a/docs/architecture/how/serialization.md +++ b/docs/architecture/how/serialization.md @@ -41,8 +41,8 @@ pub struct BlockHeader { ``` -This is done to protect structures from accidental changes that could corrupt the -database or disrupt the protocol. Dedicated CI check is responsible to check the +This is done to protect structures from accidental changes that could corrupt the +database or disrupt the protocol. Dedicated CI check is responsible to check the consistency of the schema. See [README](../../../tools/protocol-schema-check/README.md) for more details. All these structures are likely to implement BorshSerialize and BorshDeserialize (see below). @@ -59,7 +59,7 @@ You can read more on how Borsh serializes the data, by looking at the Specificat tab on [borsh.io](https://borsh.io). The biggest pitfall/risk of Borsh, is that any change to the structure, might -cause previous data to no longer be parseable. +cause previous data to no longer be parsable. For example, inserting a new enum ‘in the middle’: @@ -90,7 +90,7 @@ pub enum MyCar { ``` Is such a scenario - some of the objects created by binaries with this feature -enabled, will not be parseable by binaries without this feature. +enabled, will not be parsable by binaries without this feature. Removing and adding fields to structures is also dangerous. @@ -122,6 +122,7 @@ message HandshakeFailure { } ``` + Afterwards, such a proto file is fed to protoc ‘compiler’ that returns auto-generated code (in our case Rust code) - that can be directly imported into your library. diff --git a/docs/architecture/how/sync.md b/docs/architecture/how/sync.md index 4d66d4c4b17..ac751bf6415 100644 --- a/docs/architecture/how/sync.md +++ b/docs/architecture/how/sync.md @@ -13,7 +13,6 @@ blocks etc). tracking) additional shards in the future epochs. Currently it should be a no-op for 99% of nodes (see below). - **Tracking shards:** as you know our system has multiple shards (currently 4). Currently 99% of nodes are tracking all the shards: validators have to - as they have to validate the chunks from all the shards, and normal nodes mostly also @@ -44,7 +43,6 @@ As headers are quite small, we try to request multiple of them in a single call ### Step 1a: Epoch Sync [normal node*] // not implemented yet - While currently normal nodes are using Header sync, we could actually allow them to do something faster - “light client sync” a.k.a “epoch sync”. @@ -193,7 +191,6 @@ states are not ready, corresponding to `NotCaughtUp`, then only the shards for this epoch will be applied. When `catchup_blocks`, shards for the next epoch will be applied. - ```rust enum ApplyChunksMode { IsCaughtUp, @@ -220,20 +217,19 @@ is caught up and if we need to download states. The logic works as follows: * For other blocks, we mark them as not caught up if the previous block is not caught up. This info is persisted in `DBCol::BlocksToCatchup` which stores mapping from previous block to vector of all child blocks to catch up. -* Chunks for already tracked shards will be applied during `process_block`, as +* Chunks for already tracked shards will be applied during `process_block`, as we said before mentioning `ApplyChunksMode`. -* Once we downloaded state, we start catchup. It will take blocks from - `DBCol::BlocksToCatchup` in breadth-first search order and apply chunks for +* Once we downloaded state, we start catchup. It will take blocks from + `DBCol::BlocksToCatchup` in breadth-first search order and apply chunks for shards which have to be tracked in the next epoch. * When catchup doesn't see any more blocks to process, `DBCol::BlocksToCatchup` is cleared, which means that catchup process is finished. - The catchup process is implemented through the function `Client::run_catchup`. `ClientActor` schedules a call to `run_catchup` every 100ms. However, the call can be delayed if ClientActor has a lot of messages in its actix queue. -Every time `run_catchup` is called, it checks `DBCol::StateDlInfos` to see +Every time `run_catchup` is called, it checks `DBCol::StateDlInfos` to see if there are any shard states that should be downloaded. If so, it initiates the syncing process for these shards. After the state is downloaded, `run_catchup` will start to apply blocks that need to be caught up. diff --git a/docs/architecture/how/tx_receipts.md b/docs/architecture/how/tx_receipts.md index 584a5debe1b..c64e0bd1e5f 100644 --- a/docs/architecture/how/tx_receipts.md +++ b/docs/architecture/how/tx_receipts.md @@ -73,7 +73,7 @@ consider the transaction to be successful. ### [Advanced] But reality is more complex -**Caution:** In the section below, some things are simplified and do not match exactly +**Caution:** In the section below, some things are simplified and do not match exactly how the current code works. Let’s quickly also check what’s inside a Chunk: @@ -93,6 +93,7 @@ ones for this chunk, but instead the ‘outgoing’ ones from the previous block This has to do with performance. The steps usually followed for producing a block are as follows + 1. Chunk producer executes the receipts and creates a chunk. It sends the chunk to other validators. Note that it's the execution/processing of the receipts that usually takes the most time. 2. Validators receive the chunk and validate it before signing the chunk. Validation involves executing/processing of the receipts in the chunk. 3. Once the next block chunk producer receives the validation (signature), only then can it start producing the next chunk. diff --git a/docs/architecture/how/tx_routing.md b/docs/architecture/how/tx_routing.md index 418b755df46..0a4cfa969e7 100644 --- a/docs/architecture/how/tx_routing.md +++ b/docs/architecture/how/tx_routing.md @@ -71,7 +71,7 @@ is produced. What happens afterwards will be covered in future episodes/articles. -## Additional notes: +## Additional notes ### Transaction being added multiple times diff --git a/docs/architecture/network.md b/docs/architecture/network.md index a9fc35fcdb5..8135ce8be7b 100644 --- a/docs/architecture/network.md +++ b/docs/architecture/network.md @@ -13,7 +13,7 @@ code without any prior knowledge. # 2. Code structure -`near-network` runs on top of the `actor` framework called +`near-network` runs on top of the `actor` framework called [`Actix`](https://actix.rs/docs/). Code structure is split between 4 actors `PeerManagerActor`, `PeerActor`, `RoutingTableActor`, `EdgeValidatorActor` @@ -26,6 +26,7 @@ edge validation involves verifying cryptographic signatures, which can be quite expensive, and therefore was moved to another thread. Responsibilities: + * Validating edges by checking whenever cryptographic signatures match. ### 2.2 `RoutingTableActor` @@ -38,6 +39,7 @@ through a `TCP connection`. Otherwise, `RoutingTableActor` is responsible for pi the best path between them. Responsibilities: + * Keep set of all edges of `P2P network` called routing table. * Connects to `EdgeValidatorActor`, and asks for edges to be validated, when needed. @@ -50,6 +52,7 @@ created. Each `PeerActor` keeps a physical `TCP connection` to exactly one peer. Responsibilities: + * Maintaining physical connection. * Reading messages from peers, decoding them, and then forwarding them to the right place. @@ -68,6 +71,7 @@ through this `Actor`. `PeerManagerActor` is responsible for accepting incoming connections from the outside world and creating `PeerActors` to manage them. Responsibilities: + * Accepting new connections. * Maintaining the list of `PeerActors`, creating, deleting them. * Routing information about new edges between `PeerActors` and @@ -90,12 +94,13 @@ on its own thread. config`. Here is a list of features read from config: + * `boot_nodes` - list of nodes to connect to on start. * `addr` - listening address. * `max_num_peers` - by default we connect up to 40 peers, current implementation supports up to 128. -# 5. Connecting to other peers. +# 5. Connecting to other peers Each peer maintains a list of known peers. They are stored in the database. If the database is empty, the list of peers, called boot nodes, will be read from @@ -115,6 +120,7 @@ Both are defined below. # 6.1 PublicKey We use two types of public keys: + * a 256 bit `ED25519` public key. * a 512 bit `Secp256K1` public key. @@ -142,6 +148,7 @@ pub struct PeerId(PublicKey); # 6.3 Edge Each `edge` is represented by the `Edge` structure. It contains the following: + * pair of nodes represented by their public keys. * `nonce` - a unique number representing the state of an edge. Starting with `1`. Odd numbers represent an active edge. Even numbers represent an edge in which @@ -229,6 +236,7 @@ pub struct Handshake { Node `B` receives a `Handshake` message. Then it performs various validation checks. That includes: + * Check signature of edge from the other peer. * Whenever `nonce` is the edge, send matches. * Check whether the protocol is above the minimum @@ -337,6 +345,7 @@ This message is then forwarded to `RoutingTableActor`. the `ValidateEdgeList` struct. `ValidateEdgeList` contains: + * list of edges to verify. * peer who sent us the edges. @@ -384,6 +393,7 @@ a recalculation (see `RoutingTableActor::needs_routing_table_recalculation`). # 9 Routing table computation Routing table computation does a few things: + * For each peer `B`, calculates set of peers `|C_b|`, such that each peer is on the shortest path to `B`. * Removes unreachable edges from memory and stores them to disk. @@ -403,6 +413,7 @@ Routing table computation does a few things: ## 9.2 Step 2 `RoutingTableActor` receives the message, and then: + * calls `recalculate_routing_table` method, which computes `RoutingTableActor::peer_forwarding: HashMap>`. For each `PeerId` on the network, gives a list of connected peers, which are on the @@ -419,26 +430,27 @@ Routing table computation does a few things: `PeerManagerActor` keeps a local copy of `edges_info`, called `local_edges_info` containing only edges adjacent to current node. + * `RoutingTableUpdateResponse` contains a list of local edges, which `PeerManagerActor` should remove. * `peer_forwarding` which represents how to route messages in the P2P network * `peers_to_ban` represents a list of peers to ban for sending us edges, which failed validation in `EdgeVerifierActor`. - ## 9.4 Step 4 `PeerManagerActor` receives `RoutingTableUpdateResponse` and then: + * updates local copy of `peer_forwarding`, used for routing messages. * removes `local_edges_to_remove` from `local_edges_info`. * bans peers, who sent us invalid edges. -# 10. Message transportation layers. +# 10. Message transportation layers This section describes different protocols of sending messages currently used in `Near`. -## 10.1 Messages between Actors. +## 10.1 Messages between Actors `Near` is built on `Actix`'s `actor` [framework](https://actix.rs/docs/actix/actor). Usually each actor @@ -480,7 +492,7 @@ and convert into RoutedMessage (which also have things like TTL etc.). Then it will use the `routing_table`, to find the route to the target peer (add `route_back` if needed) and then send the message over the network as `PeerMessage::Routed`. Details about routing table computations are covered in -[section 8](#8-adding-new-edges-to-routing-tables). +[section 8](#8-adding-new-edges-to-routing-tables). When Peer receives this message (as `PeerMessage::Routed`), it will pass it to PeerManager (as `RoutedMessageFrom`), which would then check if the message is @@ -497,7 +509,7 @@ messages that are meant to be sent to another `peer`. `lib.rs` (`ShardsManager`) has a `network_adapter` - coming from the client’s `network_adapter` that comes from `ClientActor` that comes from the `start_client` call that comes from `start_with_config` (that creates `PeerManagerActor` - that is -passed as target to `network_recipent`). +passed as target to `network_recipient`). # 12. Database @@ -513,6 +525,7 @@ Each component is assigned a unique `nonce`, where first one is assigned nonce 0. Each new component gets assigned a consecutive integer. To store components, we have the following columns in the DB. + * `DBCol::LastComponentNonce` Stores `component_nonce: u64`, which is the last used nonce. * `DBCol::ComponentEdges` Mapping from `component_nonce` to a list of edges. @@ -520,5 +533,5 @@ To store components, we have the following columns in the DB. ### 12.2 Storage of `account_id` to `peer_id` mapping -`ColAccountAnouncements` -> Stores a mapping from `account_id` to a tuple +`ColAccountAnnouncements` -> Stores a mapping from `account_id` to a tuple (`account_id`, `peer_id`, `epoch_id`, `signature`). diff --git a/docs/architecture/next/README.md b/docs/architecture/next/README.md index 867a4cd09d4..90b8fdba96b 100644 --- a/docs/architecture/next/README.md +++ b/docs/architecture/next/README.md @@ -1,4 +1,5 @@ # How neard will work + The documents under this chapter are talking about the future of NEAR - what we're planning on improving and how. (This also means that they can get out of date quickly :-). diff --git a/docs/architecture/next/catchup_and_state_sync.md b/docs/architecture/next/catchup_and_state_sync.md index ac5d2a45068..0075a60885b 100644 --- a/docs/architecture/next/catchup_and_state_sync.md +++ b/docs/architecture/next/catchup_and_state_sync.md @@ -1,13 +1,10 @@ This document is still a DRAFT. - - This document covers our improvement plans for state sync and catchup. **Before reading this doc, you should take a look at [How sync works](../how/sync.md)** - - State sync is used in two situations: + * when your node is behind for more than 2 epochs (and it is not an archival node) - then rather than trying to apply block by block (that can take hours) - you 'give up' and download the fresh state (a.k.a state sync) and apply blocks from there. * when you're a block (or chunk) producer - and in the upcoming epoch, you'll have to track a shard that you are not currently tracking. @@ -16,16 +13,18 @@ In the past (and currently) - the state sync was mostly used in the first scenar As we progress towards phase 2 and keep increasing number of shards - the catchup part starts being a lot more critical. When we're running a network with a 100 shards, the single machine is simply not capable of tracking (a.k.a applying all transactions) of all shards - so it will have to track just a subset. And it will have to change this subset almost every epoch (as protocol rebalances the shard-to-producer assignment based on the stakes). This means that we have to do some larger changes to the state sync design, as requirements start to differ a lot: + * catchups are high priority (the validator MUST catchup within 1 epoch - otherwise it will not be able to produce blocks for the new shards in the next epoch - and therefore it will not earn rewards). * a lot more catchups in progress (with lots of shards basically every validator would have to catchup at least one shard at each epoch boundary) - this leads to a lot more potential traffic on the network -* malicious attacks & incentives - the state data can be large and can cause a lot of network traffic. At the same time it is quite critical (see point above), so we'll have to make sure that the nodes are incentivised to provide the state parts upon request. +* malicious attacks & incentives - the state data can be large and can cause a lot of network traffic. At the same time it is quite critical (see point above), so we'll have to make sure that the nodes are incentivize to provide the state parts upon request. * only a subset of peers will be available to request the state sync from (as not everyone from our peers will be tracking the shard that we're interested in). - -## Things that we're actively analysing +## Things that we're actively analyzing ### Performance of state sync on the receiver side + We're looking at the performance of state sync: + * how long does it take to create the parts, * pro-actively creating the parts as soon as epoch starts * creating them in parallel @@ -41,19 +40,19 @@ When we receive a part, we should announce this information to our peers - so th ## Ideas - not actively working on them yet ### Better networking (a.k.a Tier 3) + Currently our networking code is picking the peers to connect at random (as most of them are tracking all the shards). With phase2 it will no longer be the case, so we should work on improvements of our peer-selection mechanism. In general - we should make sure that we have direct connection to at least a few nodes that are tracking the same shards that we're tracking right now (or that we'll want to track in the near future). ### Dedicated nodes optimized towards state sync responses + The idea is to create a set of nodes that would specialize in state sync responses (similar to how we have archival nodes today). The sub-idea of this, is to store such data on one of the cloud providers (AWS, GCP). ### Sending deltas instead of full state syncs + In case of catchup, the requesting node might have tracked that shard in the past. So we could consider just sending a delta of the state rather than the whole state. While this helps with the amount of data being sent - it might require the receiver to do a lot more work (as the data that it is about to send cannot be easily cached). - - - diff --git a/docs/architecture/next/malicious_chunk_producer_and_phase2.md b/docs/architecture/next/malicious_chunk_producer_and_phase2.md index a42ae97175c..2def6c6b137 100644 --- a/docs/architecture/next/malicious_chunk_producer_and_phase2.md +++ b/docs/architecture/next/malicious_chunk_producer_and_phase2.md @@ -1,23 +1,23 @@ -# Malicious producers in phase 2 of sharding. +# Malicious producers in phase 2 of sharding In this document, we'll compare the impact of the hypothetical malicious producer on the NEAR system (both in the current setup and how it will work when phase2 is implemented). ## Current state (Phase 1) -Let's assume that a malicious chunk producer ``C1`` has produced a bad chunk -and sent it to the block producer at this height ``B1``. +Let's assume that a malicious chunk producer ``C1`` has produced a bad chunk +and sent it to the block producer at this height ``B1``. -The block producer IS going to add the chunk to the block (as we don't validate -the chunks before adding to blocks - but only when signing the block - see +The block producer IS going to add the chunk to the block (as we don't validate +the chunks before adding to blocks - but only when signing the block - see [Transactions and receipts - last section](./../how/tx_receipts.md)). -After this block is produced, it is sent to all the validators to get the +After this block is produced, it is sent to all the validators to get the signatures. -As currently all the validators are tracking all the shards - they will quickly +As currently all the validators are tracking all the shards - they will quickly notice that the chunk is invalid, so they will not sign the block. -Therefore the next block producer ``B2`` is going to ignore ``B1``'s block, and +Therefore the next block producer ``B2`` is going to ignore ``B1``'s block, and select block from ``B0`` as a parent instead. So TL;DR - **a bad chunk would not be added to the chain.** @@ -26,15 +26,15 @@ So TL;DR - **a bad chunk would not be added to the chain.** Unfortunately things get a lot more complicated, once we scale. -Let's assume the same setup as above (a single chunk producer ``C1`` being -malicious). But this time, we have 100 shards - each validator is tracking just -a few (they cannot track all - as today - as they would have to run super +Let's assume the same setup as above (a single chunk producer ``C1`` being +malicious). But this time, we have 100 shards - each validator is tracking just +a few (they cannot track all - as today - as they would have to run super powerful machines with > 100 cores). -So in the similar scenario as above - ``C1`` creates a malicious chunks, and +So in the similar scenario as above - ``C1`` creates a malicious chunks, and sends it to ``B1``, which includes it in the block. -And here's where the complexity starts - as most of the validators will NOT +And here's where the complexity starts - as most of the validators will NOT track the shard which ``C1`` was producing - so they will still sign the block. The validators that do track that shard will of course (assuming that they are non-malicious) refuse the sign. But overall, they will be a small majority - so the block is going to get enough signatures and be added to the chain. @@ -47,26 +47,25 @@ Challenges, Slashing and Rollbacks. #### Challenge -Challenge is a self-contained proof, that something went wrong in the chunk +Challenge is a self-contained proof, that something went wrong in the chunk processing. It must contain all the inputs (with their merkle proof), the code that was executed, and the outputs (also with merkle proofs). -Such a challenge allows anyone (even nodes that don't track that shard or have +Such a challenge allows anyone (even nodes that don't track that shard or have any state) to verify the validity of the challenge. -When anyone notices that a current chain contains a wrong transition - they -submit such challenge to the next block producer, which can easily verify it +When anyone notices that a current chain contains a wrong transition - they +submit such challenge to the next block producer, which can easily verify it and it to the next block. -Then the validators do the verification themselves, and if successful, they +Then the validators do the verification themselves, and if successful, they sign the block. -When such block is successfully signed, the protocol automatically slashes -malicious nodes (more details below) and initiates the rollback to bring the -state back to the state before the bad chunk (so in our case, back to the block +When such block is successfully signed, the protocol automatically slashes +malicious nodes (more details below) and initiates the rollback to bring the +state back to the state before the bad chunk (so in our case, back to the block produced by `B0`). - #### Slashing Slashing is the process of taking away the part of the stake from validators @@ -75,6 +74,7 @@ that are considered malicious. In the example above, we'll definitely need to slash the ``C1`` - and potentially also any validators that were tracking that shard and did sign the bad block. Things that we'll have to figure out in the future: + * how much do we slash? all of the stake? some part? * what happens to the slashed stake? is it burned? does it go to some pool? @@ -82,27 +82,26 @@ Things that we'll have to figure out in the future: // TODO: add - ## Problems with the current Phase 2 design ### Is slashing painful enough? + In the example above, we'd successfully slash the ``C1`` producer - but was it enough? -Currently (with 4 shards) you need around 20k NEAR to become a chunk producer. -If we increase the number of shards to 100, it would drop the minimum stake to +Currently (with 4 shards) you need around 20k NEAR to become a chunk producer. +If we increase the number of shards to 100, it would drop the minimum stake to around 1k NEAR. -In such scenario, by sacrificing 1k NEAR, the malicious node can cause the -system to rollback a couple blocks (potentially having bad impact on the bridge +In such scenario, by sacrificing 1k NEAR, the malicious node can cause the +system to rollback a couple blocks (potentially having bad impact on the bridge contracts etc). -On the other side, you could be a non-malicious chunk producer with a corrupted -database (or a nasty bug in the code) - and the effect would be the same - the -chunk that you produced would be marked as malicious, and you'd lose your stake +On the other side, you could be a non-malicious chunk producer with a corrupted +database (or a nasty bug in the code) - and the effect would be the same - the +chunk that you produced would be marked as malicious, and you'd lose your stake (which will be a super-scary even for any legitimate validator). - So the open question is - can we do something 'smarter' in the protocol to -detect the case, where there is 'just a single' malicious (or buggy) chunk +detect the case, where there is 'just a single' malicious (or buggy) chunk producer and avoid the expensive rollback? diff --git a/docs/architecture/storage.md b/docs/architecture/storage.md index 5ae0e2d49be..042328b63a2 100644 --- a/docs/architecture/storage.md +++ b/docs/architecture/storage.md @@ -1,4 +1,4 @@ # Storage This is our work-in-progress storage documentation. Things are raw and -incomplete. You are encouraged to help improve it, in any capacity you can! \ No newline at end of file +incomplete. You are encouraged to help improve it, in any capacity you can! diff --git a/docs/architecture/storage/README.md b/docs/architecture/storage/README.md index baf06d02755..dba726d0913 100644 --- a/docs/architecture/storage/README.md +++ b/docs/architecture/storage/README.md @@ -3,10 +3,11 @@ ## Overview The storage subsystem of nearcore is complex and has many layers. This documentation -provides an overview of the common usecases and explains the most important +provides an overview of the common use cases and explains the most important implementation details. The main requirements are: + - Low but predictable latency for reads - Proof generation for chunk validators @@ -33,17 +34,17 @@ When RPC or validator node receives a transaction, it needs to do validity check It involves reading Accounts and AccessKeys, which is also a storage operation. Another query to RPC node is a view call - state query without modification or -contract dry run. +contract dry run. ## Focus In the documentation, we focus the most on the **Contract Storage** use case because it has the strictest requirements. -For the high-level flow, refer to [Flow diagram](./flow.md). +For the high-level flow, refer to [Flow diagram](./flow.md). For more detailed information, refer to: -* [Primitives](./primitives.md) -* [Trie Storage](./trie_storage.md) -* [Flat Storage](./flat_storage.md) -* [Database](./database.md) +- [Primitives](./primitives.md) +- [Trie Storage](./trie_storage.md) +- [Flat Storage](./flat_storage.md) +- [Database](./database.md) diff --git a/docs/architecture/storage/database.md b/docs/architecture/storage/database.md index 3642dab9fb9..70cfdb2afae 100644 --- a/docs/architecture/storage/database.md +++ b/docs/architecture/storage/database.md @@ -16,9 +16,9 @@ We store the database in RocksDB. This document is an attempt to give hints abou - The state changes are stored in column family `StateChanges`, number 35 - In this family, each key is of the form `BlockHash | Column | AdditionalInfo` where: - + `BlockHash: [u8; 32]` is the block hash for this change - + `Column: u8` is defined near the top of `core/primitives/src/trie_key.rs` - + `AdditionalInfo` depends on `Column` and it can be found in the code for the `TrieKey` struct, same file as `Column` + - `BlockHash: [u8; 32]` is the block hash for this change + - `Column: u8` is defined near the top of `core/primitives/src/trie_key.rs` + - `AdditionalInfo` depends on `Column` and it can be found in the code for the `TrieKey` struct, same file as `Column` ### Contract Deployments @@ -26,6 +26,7 @@ We store the database in RocksDB. This document is an attempt to give hints abou - `AdditionalInfo` is the account id for which the contract is being deployed - The key value contains the contract code alongside other pieces of data. It is possible to extract the contract code by removing everything until the wasm magic number, 0061736D01000000 - As such, it is possible to dump all the contracts that were ever deployed on-chain using this command on an archival node: + ``` ldb --db=~/.near/data scan --column_family=col35 --hex | \ grep -E '^0x.{64}01' | \ @@ -33,5 +34,6 @@ We store the database in RocksDB. This document is an attempt to give hints abou sed 's/^.*x/0061736D01000000/' | \ grep -v ' : ' ``` + (Note that the last grep is required because not every such value appears to contain contract code) We should implement a feature to state-viewer that’d allow better visualization of this data, but in the meantime this seems to work. diff --git a/docs/architecture/storage/flat_storage.md b/docs/architecture/storage/flat_storage.md index f1430ae6dc5..83f581f81d7 100644 --- a/docs/architecture/storage/flat_storage.md +++ b/docs/architecture/storage/flat_storage.md @@ -8,7 +8,7 @@ content-addressed storage system that can easily self-verify. Flat storage is a bit like a database index for the values stored in the trie. It stores a copy of the data in a more accessible way to speed up the lookup -time. +time. Drastically oversimplified, flat storage uses a hashmap instead of a trie. This reduces the lookup time from `O(d)` to `O(1)` where `d` is the tree depth of the @@ -83,11 +83,11 @@ the trie, plus one more for dereferencing the final value. Clearly, `d + 1` is worse than `1 + 1`, right? Well, as it turns out, we can cache the trie nodes with surprisingly high effectiveness. In mainnet workloads (which were often optimized to work well with the trie shape) we -observe a 99% cache hit rate in many cases. +observe a 99% cache hit rate in many cases. Combine that with the fact that a typical value for `d` is somewhere between 10 and 20. Then we may conclude that, in expectation, a trie lookup (`d * 0.01 + 1` -requests) requires less DB requests than a flat state lookup (`1 + 1` requests). +requests) requires less DB requests than a flat state lookup (`1 + 1` requests). In practice, however, flat state still has an edge over accessing trie storage directly. And that is due to two reasons. @@ -171,12 +171,12 @@ Here we describe structures used for flat storage implementation. ## FlatStorage -This is the main structure which owns information about ValueRefs for all keys from some fixed +This is the main structure which owns information about ValueRefs for all keys from some fixed shard for some set of blocks. It is shared by multiple threads, so it is guarded by RwLock: * Chain thread, because it sends queries like: - * "new block B was processed by chain" - supported by add_block - * "flat storage head can be moved forward to block B" - supported by update_flat_head + * "new block B was processed by chain" - supported by add_block + * "flat storage head can be moved forward to block B" - supported by update_flat_head * Thread that applies a chunk, because it sends read queries "what is the ValueRef for key for block B" * View client (not fully decided) @@ -203,14 +203,14 @@ by get_ref method. If storage is fully empty, then we need to create flat storage from scratch. FlatStorage is stored inside NightshadeRuntime, and it itself is stored inside Chain, so we need to create them in the same order and dependency hierarchy should be the same. But at the same time, we parse genesis file only during Chain -creation. That’s why FlatStorageManager has set_flat_storage_for_genesis method which is called +creation. That’s why FlatStorageManager has set_flat_storage_for_genesis method which is called during Chain creation. ### Regular block processing vs. catchups -For these two usecases we have two different flows: first one is handled in Chain.postprocess_block, +For these two use cases we have two different flows: first one is handled in Chain.postprocess_block, the second one in Chain.block_catch_up_postprocess. Both, when results of applying chunk are ready, -should call Chain.process_apply_chunk_result → RuntimeAdapter.get_flat_storage_for_shard → +should call Chain.process_apply_chunk_result → RuntimeAdapter.get_flat_storage_for_shard → FlatStorage.add_block, and when results of applying ALL processed/postprocessed chunks are ready, should call RuntimeAdapter.get_flat_storage_for_shard → FlatStorage.update_flat_head. diff --git a/docs/architecture/storage/flow.md b/docs/architecture/storage/flow.md index 99b492ad222..f171536887e 100644 --- a/docs/architecture/storage/flow.md +++ b/docs/architecture/storage/flow.md @@ -9,6 +9,7 @@ Blue arrow means a call triggered by read. Red arrow means a call triggered by write. Black arrow means a non-trivial data dependency. For example: + * Nodes which are read on TrieStorage go to TrieRecorder to generate proof, so they are connected with black arrow. * Memtrie lookup needs current state of accounting cache to compute costs. When diff --git a/docs/architecture/storage/primitives.md b/docs/architecture/storage/primitives.md index fa4f40d1329..b000826f875 100644 --- a/docs/architecture/storage/primitives.md +++ b/docs/architecture/storage/primitives.md @@ -98,4 +98,4 @@ to `Option`. Implementation depends on the main storage sourc Expected side effects are * changing accessed node counters so that contract runtime can charge caller based on them, -* recording accessed nodes for a proof for chunk validators. \ No newline at end of file +* recording accessed nodes for a proof for chunk validators. diff --git a/docs/architecture/storage/trie_storage.md b/docs/architecture/storage/trie_storage.md index 9a7162a048b..76e7429ff89 100644 --- a/docs/architecture/storage/trie_storage.md +++ b/docs/architecture/storage/trie_storage.md @@ -16,7 +16,7 @@ Trie stores the state - accounts, contract codes, access keys, etc. Each state item corresponds to the unique trie key. You can read more about this structure on [Wikipedia](https://en.wikipedia.org/wiki/Trie). -There are two ways to access trie - from memory and from disk. The first one is +There are two ways to access trie - from memory and from disk. The first one is currently the main one, where only the loading stage requires disk, and the operations are fully done in memory. The latter one relies only on disk with several layers of caching. Here we describe the disk trie. @@ -107,7 +107,7 @@ collection of old block data which is no longer needed. ### TrieRefcountChange Because we remove unused nodes during garbage collection, we need to track -the reference count (`rc`) for each node. Another reason is that we can dedup +the reference count (`rc`) for each node. Another reason is that we can de-dup values. If the same contract is deployed 1000 times, we only store one contract binary in storage and track its count. diff --git a/docs/misc/archival_data_recovery.md b/docs/misc/archival_data_recovery.md index 11af068f327..059075cd53f 100644 --- a/docs/misc/archival_data_recovery.md +++ b/docs/misc/archival_data_recovery.md @@ -10,7 +10,7 @@ The practical effect of this issue is that requests querying the state of an acc The simplest way to check whether or not a node has suffered data loss is to run one of the following queries: -*replace with the correct URL (example: http://localhost:3030)* +*replace with the correct URL (example: )* ```bash curl -X POST \ @@ -26,6 +26,7 @@ curl -X POST \ { "id": "dontcare", "jsonrpc": "2.0", "method": "query", "params": { "account_id": "token2.near", "block_id": 114580308, "request_type": "view_account" } }' ``` + ```bash curl -X POST \ -H "Content-Type: application/json" \ @@ -46,14 +47,13 @@ If for any of the above requests you get an error of the kind `MissingTrieValue` ## Option A (recommended): download a new DB snapshot - 1. Stop `neard` 2. Delete the existing `hot` and `cold` databases. Example assuming default configuration: + ```bash rm -rf ~/.near/hot-data && rm -rf ~/.near/cold-data ``` - 3. Download an up-to-date snapshot, following this guide: [Storage snapshots](https://near-nodes.io/archival/split-storage-archival#S3%20migration) ## Option B: manually run recovery commands @@ -119,4 +119,4 @@ RUST_LOG=debug neard database resharding --height 115185107 --shard-id 4 --resto ## Verify if remediation has been successful -Run the queries specified in the section: [Check if my node has been impacted](https://docs.nearone.org/doc/archival-node-recovery-of-missing-data-speQFTJc0L#h-check-if-my-node-has-been-impacted). All of them should return a successful response now. \ No newline at end of file +Run the queries specified in the section: [Check if my node has been impacted](https://docs.nearone.org/doc/archival-node-recovery-of-missing-data-speQFTJc0L#h-check-if-my-node-has-been-impacted). All of them should return a successful response now. diff --git a/docs/misc/contract_distribution.md b/docs/misc/contract_distribution.md index 4bacf778100..ad2e6b0b8c0 100644 --- a/docs/misc/contract_distribution.md +++ b/docs/misc/contract_distribution.md @@ -1,8 +1,9 @@ # Distributing contracts separately from state witness -The current chunk state witness structure is inefficient due to a significant portion being consumed by contract code. This document describes an implementation for optimizing the state witness size. The changes described are guarded by the protocol feature `ExcludeContractCodeFromStateWitness`. +The current chunk state witness structure is inefficient due to a significant portion being consumed by contract code. This document describes an implementation for optimizing the state witness size. The changes described are guarded by the protocol feature `ExcludeContractCodeFromStateWitness`. In feature `ExcludeContractCodeFromStateWitness`, we optimize the state witness size by distributing the contract code separately from the witness, under the following observations: + 1. New contracts are deployed infrequently, so the function calls are often made to the same contract and we distribute the same contract code in the witness many times at different heights. 2. Once deployed, a contract is compiled and stored in the compiled-contract cache. This cache is stored in the disk and persistent across epochs, until the VM configuration changes. The chunk application uses this cache to bypass the trie traversal and storage reads to fetch the uncompiled contract code from storage. The cache is maintained with high hit rates by both chunk producers and chunk validators. @@ -23,19 +24,20 @@ A `FunctionCallAction` represents calling a method of the contract deployed to a The function call is fulfilled by either retrieving the pre-compiled contract from the cache or compiling the code right before execution. Note that, not all chunk validators may contain the contract in their compiled-contract cache. -Thus, the contract code (which is also part of the State) should be available when validating the state witness. +Thus, the contract code (which is also part of the State) should be available when validating the state witness. To address this, independent of whether the compiled contract cache was hit or not, all the accesses to the uncompiled code are recorded by [explicitly reading](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/core/store/src/trie/mod.rs#L1628-L1637) from the key `TrieKey::ContractCode{account_id}`, which records all the internal trie nodes from the state root upto the leaf node and the value (code) itself. Thus, each function call will contribute to the state witness size at least the contract code, which may go up to `4 MB`. ## Excluding contract code from state witness When `ExcludeContractCodeFromStateWitness` is enabled, we distribute the following in parallel to the state witness: + 1. Hashes of the contracts code called during the chunk application. We do not include the contract code in this message. Instead, the chunk validators request the contracts missing in their compiled-contract cache. 1. Code of the contracts deployed during the chunk application. We distribute this message only to the validators other than the chunk validators, since the chunk validators can access the new contract code and update their cache by applying the deploy actions (contained in the incoming receipts) in the state witness. ### Collecting contract accesses and deployments -In order to identify which contracts to distribute, we collect (1) the hashes of the contracts called by a `FunctionCallAction` and (2) contract code deployed by a `DeployContractAction`. +In order to identify which contracts to distribute, we collect (1) the hashes of the contracts called by a `FunctionCallAction` and (2) contract code deployed by a `DeployContractAction`. When `ExcludeContractCodeFromStateWitness` is enabled, the chunk producer performs the following when applying the receipts in a chunk (note that it is done by all the chunk producers tracking the same shard): - For function calls, it skips recoding the read of the value from `TrieKey::ContractCode{account_id}`. Instead, it just records the hash of the contract code. [The `TrieUpdate::record_contract_call` function](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/core/store/src/trie/update.rs#L267) called when executing a `FunctionCallAction` implements the different behaviors with and without the feature enabled. @@ -51,9 +53,10 @@ Upon finishing producing the new chunk, the chunk producer reconstructs `Contrac NOTE: All the operations described in the rest of this document are performed in the `PartialWitnessActor`. `PartialWitnessActor` distributes the state witness and the contract updates in the following order ([see code here](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/chain/client/src/stateless_validation/partial_witness/partial_witness_actor.rs#L207-L246)): + 1. It first sends the hashes of the contract code accessed to the chunk validators (except for the validators that track the same shard). This allows validators to check their compiled-contract cache and request code for the missing contracts, while waiting for the witness parts. This is sent in a message called `ChunkContractAccesses`. 1. It then send the state witness parts to witness-part owners. -1. It finally sends the new contracts deployed to the validators that do not validate the witness in the current turn. This allows the other validators to update their compiled-contract cache for the later turns when they become a chunk validator for the respective shard. The parts are sent in a message called `PartialEncodedContractDeploys`. The code for deployed contracts is distributed to validators in parts after compressing and encoding in `Reed-Solomon code`, similarly to how the state witness is sent in parts. +1. It finally sends the new contracts deployed to the validators that do not validate the witness in the current turn. This allows the other validators to update their compiled-contract cache for the later turns when they become a chunk validator for the respective shard. The parts are sent in a message called `PartialEncodedContractDeploys`. The code for deployed contracts is distributed to validators in parts after compressing and encoding in `Reed-Solomon code`, similarly to how the state witness is sent in parts. ### Handling contract accesses messages in chunk validators @@ -64,4 +67,4 @@ A chunk producer receiving the `ContractCodeRequest` validates the request again ### Handling deployed code messages in other validators -When a validator receives a `PartialEncodedContractDeploys` message for a newly deployed contract, it starts collecting the parts using its [`PartialEncodedContractDeploysTracker`](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/chain/client/src/stateless_validation/partial_witness/partial_deploys_tracker.rs#L74). The tracker waits until the sufficient number of parts are collected, then decodes and decompresses the original contracts and [compiles them in parallel](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/chain/client/src/stateless_validation/partial_witness/partial_witness_actor.rs#L522). The compilation internally persists the compiled contract code in the local cache, so that the validator can use the compiled code later when validating the upcoming chunk witnesses. \ No newline at end of file +When a validator receives a `PartialEncodedContractDeploys` message for a newly deployed contract, it starts collecting the parts using its [`PartialEncodedContractDeploysTracker`](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/chain/client/src/stateless_validation/partial_witness/partial_deploys_tracker.rs#L74). The tracker waits until the sufficient number of parts are collected, then decodes and decompresses the original contracts and [compiles them in parallel](https://github.com/near/nearcore/blob/82707e8edfd1af7b1d2e5bb1c82ccf768c313e7c/chain/client/src/stateless_validation/partial_witness/partial_witness_actor.rs#L522). The compilation internally persists the compiled contract code in the local cache, so that the validator can use the compiled code later when validating the upcoming chunk witnesses. diff --git a/docs/misc/state_sync_dump.md b/docs/misc/state_sync_dump.md index 3e5ee4fa2a0..1801926b5d8 100644 --- a/docs/misc/state_sync_dump.md +++ b/docs/misc/state_sync_dump.md @@ -7,6 +7,7 @@ reworked. A new version is available for experimental use. This version gets state parts from external storage. The following kinds of external storage are supported: + * Local filesystem * Google Cloud Storage * Amazon S3 @@ -24,6 +25,7 @@ State Sync from External Storage. In case you would like to manage your own dumps of State, keep reading. ### Google Cloud Storage + To enable Google Cloud Storage as your external storage, add this to your `config.json` file: @@ -41,11 +43,13 @@ To enable Google Cloud Storage as your external storage, add this to your And run your node with an environment variable `SERVICE_ACCOUNT` or `GOOGLE_APPLICATION_CREDENTIALS` pointing to the credentials json file + ```shell SERVICE_ACCOUNT=/path/to/file ./neard run ``` ### Amazon S3 + To enable Amazon S3 as your external storage, add this to your `config.json` file: @@ -64,6 +68,7 @@ file: And run your node with environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`: + ```shell AWS_ACCESS_KEY_ID="MY_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="MY_AWS_SECRET_ACCESS_KEY" ./neard run ``` @@ -87,6 +92,7 @@ filesystem: In this case you don't need any extra environment variables. Simply run your node: + ```shell ./neard run ``` diff --git a/docs/misc/state_sync_from_external_storage.md b/docs/misc/state_sync_from_external_storage.md index 0986dd6a58b..b2e78449418 100644 --- a/docs/misc/state_sync_from_external_storage.md +++ b/docs/misc/state_sync_from_external_storage.md @@ -7,6 +7,7 @@ reworked. A new version is available for experimental use. This version gets state parts from external storage. The following kinds of external storage are supported: + * Local filesystem * Google Cloud Storage * Amazon S3 @@ -23,6 +24,7 @@ storage and configures it to get state parts from a location managed by Pagoda. Note: to obtain the reference configuration file, download it by running the following command: + ```shell neard init --chain-id --download-config --download-genesis ``` @@ -58,6 +60,7 @@ your `config.json` file: ``` Then run the `neard` binary and it will access GCS anonymously: + ```shell ./neard run ``` @@ -68,7 +71,7 @@ The options suggested above will most likely work fine. * `num_concurrent_requests` determines the number of state parts across all shards that can be downloaded in parallel during state sync. -* `num_concurrent_requests_during_catchup` determines the number of state parts +* `num_concurrent_requests_during_catchup` determines the number of state parts across all shards that can be downloaded in parallel during catchup. Generally, this number should not be higher than `num_concurrent_requests`. Keep it reasonably low to allow the node to process chunks of other shards. @@ -98,6 +101,7 @@ You may add the other mentioned options too. ``` Then run the `neard` binary and it will access Amazon S3 anonymously: + ```shell ./neard run ``` @@ -123,6 +127,7 @@ You may add the other mentioned options too. ``` Then run the `neard` binary: + ```shell ./neard run ``` diff --git a/docs/misc/state_witness_size_limits.md b/docs/misc/state_witness_size_limits.md index f1645d5ea97..bf0ae3f8ff9 100644 --- a/docs/misc/state_witness_size_limits.md +++ b/docs/misc/state_witness_size_limits.md @@ -4,32 +4,34 @@ Some limits were introduced to keep the size of `ChunkStateWitness` reasonable. `ChunkStateWitness` contains all the incoming transactions and receipts that will be processed during chunk application and in theory a single receipt could be tens of megabytes in size. Distributing a `ChunkStateWitness` this large would be troublesome, so we limit the size and number of transactions, receipts, etc. The limits aim to keep the total uncompressed size of `ChunkStateWitness` under 17MiB. There are two types of size limits: + * Hard limit - the size must be below this limit, anything else is considered invalid * Soft limit - things are added until the limit is exceeded, after that things stop being added. The last added thing is allowed to slightly exceed the limit. The limits are: + * `max_transaction_size = 1.5 MiB` - * All transactions must be below 1.5 MiB, otherwise they'll be considered invalid and rejected. - * Previously was 4MiB, now reduced to 1.5MiB + * All transactions must be below 1.5 MiB, otherwise they'll be considered invalid and rejected. + * Previously was 4MiB, now reduced to 1.5MiB * `max_receipt_size - 4 MiB`: - * All receipts must be below 4 MiB, otherwise they'll be considered invalid and rejected. - * Previously there was no limit on receipt size. Set to 4MiB, might be reduced to 1.5MiB in the future to match the transaction limit. + * All receipts must be below 4 MiB, otherwise they'll be considered invalid and rejected. + * Previously there was no limit on receipt size. Set to 4MiB, might be reduced to 1.5MiB in the future to match the transaction limit. * `combined_transactions_size_limit - 4 MiB` - * Hard limit on total size of transactions from this and previous chunk. `ChunkStateWitness` contains transactions from two chunks, this limit applies to the sum of their sizes. + * Hard limit on total size of transactions from this and previous chunk. `ChunkStateWitness` contains transactions from two chunks, this limit applies to the sum of their sizes. * `new_transactions_validation_state_size_soft_limit - 500 KiB` - * Validating new transactions generates storage proof (recorded trie nodes), which has to be limited. Once transaction validation generates more storage proof than this limit, the chunk producer stops adding new transactions to the chunk. + * Validating new transactions generates storage proof (recorded trie nodes), which has to be limited. Once transaction validation generates more storage proof than this limit, the chunk producer stops adding new transactions to the chunk. * `per_receipt_storage_proof_size_limit - 4 MB` - * Executing a receipt generates storage proof. A single receipt is allowed to generate at most 4MB of storage proof. This is a hard limit, receipts which generate more than that will fail. + * Executing a receipt generates storage proof. A single receipt is allowed to generate at most 4MB of storage proof. This is a hard limit, receipts which generate more than that will fail. * `main_storage_proof_size_soft_limit - 4 MB` - * This is a limit on the total size of storage proof generated by receipts in one chunk. Once receipts generate more storage proof than this limit, the chunk producer stops processing receipts and moves the rest to the delayed queue. - * It's a soft limit, which means that the total size of storage proof could reach 8 MB (3.99MB + one receipt which generates 4MB of storage proof) - * Due to implementation details it's hard to find the exact amount of storage proof generated by a receipt, so an upper bound estimation is used instead. This upper bound assumes that every removal generates additional 2000 bytes of storage proof, so receipts which perform a lot of trie removals might be limited more than theoretically applicable. + * This is a limit on the total size of storage proof generated by receipts in one chunk. Once receipts generate more storage proof than this limit, the chunk producer stops processing receipts and moves the rest to the delayed queue. + * It's a soft limit, which means that the total size of storage proof could reach 8 MB (3.99MB + one receipt which generates 4MB of storage proof) + * Due to implementation details it's hard to find the exact amount of storage proof generated by a receipt, so an upper bound estimation is used instead. This upper bound assumes that every removal generates additional 2000 bytes of storage proof, so receipts which perform a lot of trie removals might be limited more than theoretically applicable. * `outgoing_receipts_usual_size_limit - 100 KiB` - * Limit on the size of outgoing receipts to another shard. Needed to keep the size of `source_receipt_proofs` small. - * On most block heights a shard isn't allowed to send receipts larger than 100 KiB to another shard. + * Limit on the size of outgoing receipts to another shard. Needed to keep the size of `source_receipt_proofs` small. + * On most block heights a shard isn't allowed to send receipts larger than 100 KiB to another shard. * `outgoing_receipts_big_size_limit - 4.5 MiB` - * On every block height there's one special "allowed shard" which is allowed to send larger receipts, up to 4.5 MiB in total. - * A receiving shard will receive receipts from `num_shards - 1` shards using the usual limit and one shard using the big limit. + * On every block height there's one special "allowed shard" which is allowed to send larger receipts, up to 4.5 MiB in total. + * A receiving shard will receive receipts from `num_shards - 1` shards using the usual limit and one shard using the big limit. In total that gives 4 MiB + 500 KiB + 8MB + 5*100 KiB + 4.5 MiB ~= 17 MiB of maximum witness size. Possibly a little more on missing chunks. diff --git a/docs/practices/docs.md b/docs/practices/docs.md index 6e072aeaa0b..5e30e69f541 100644 --- a/docs/practices/docs.md +++ b/docs/practices/docs.md @@ -58,7 +58,7 @@ To add a new page to the book: The doc itself is in vanilla markdown. To render documentation locally: - + ```console # Install mdBook $ cargo install mdbook diff --git a/docs/practices/engaging_and_effective_communication.md b/docs/practices/engaging_and_effective_communication.md index 2445c0a5bac..f3605fc0563 100644 --- a/docs/practices/engaging_and_effective_communication.md +++ b/docs/practices/engaging_and_effective_communication.md @@ -1,11 +1,10 @@ ## Goal of this doc -We rely heavily on asynchronous conversation for our day-to-day work via Zulip/Slack/Github. As a result, how we communicate on these asynchronous collaboration tools plays a critical role on our productivity and efficiency. +We rely heavily on asynchronous conversation for our day-to-day work via Zulip/Slack/Github. As a result, how we communicate on these asynchronous collaboration tools plays a critical role on our productivity and efficiency. In this doc, we go over two scenarios we often face in our daily interaction, list outstanding problems and propose simple ways to resolve them. The proposal is not set in stone and any suggestions and criticisms are welcome as always. - -## TL;DR; +## TL;DR Before jumping into scenarios, here is TL;DR; of the proposal. @@ -13,36 +12,39 @@ Before jumping into scenarios, here is TL;DR; of the proposal. * **Be explicit about ‘whom you want to get feedback from’**: All of us are busy with our own works and given a limited time, it is not feasible to check out every single on-going conversation. By tagging specific people, you not only help other people spend their time on something more relevant to them, but also can get the response faster. Besides, this is yet another way to send the gesture that their opinions are appreciated. -* **As a topic owner, help facilitate the conversation**: +* **As a topic owner, help facilitate the conversation**: * Make sure the thread is razor focused on the main topic and cut the potential detour as early as possible. If a new discussion point is worth continuing, move the relevant messages to a new topic so the discussion can continue in a different channel. * Occasionally summarize the thread so everyone can get up to speed quickly * Set the timeline expectation on how you want to make a progress in the discussion and guide people accordingly. * Once conclusion is reached, share it and resolve the topic. Do not leave it open unnecessarily. - ## Scenario based deep-dive -### Scenario 1. Walnut wants to share updates/announcements/news with team members. +### Scenario 1. Walnut wants to share updates/announcements/news with team members Walnut sends a message on Zulip/Slack. *“Hello everyone! we have a new guideline on our engineering practice and want to make sure everyone have a look. Here is the link! Don’t hesitate to leave a comment if you have any suggestion/question. Thanks!”* After seeing the message, team members opens the link and read the doc. As everything looks good, no one leaves a comment. + * **Potential issue**: As none of the tool we use has a way for Walnut to figure out who read their message, Walnut starts wondering whether their message was delivered. * **Proposal**: Leave a comment or add an emoji to ‘ack’ the message. -### Scenario 2. Nala has an idea and wants to have technical discussion with other engineers to collect feedback and reach to the conclusion. + +### Scenario 2. Nala has an idea and wants to have technical discussion with other engineers to collect feedback and reach to the conclusion Nala starts a new topic on Zulip regarding a storage issue. *“I have a new idea to address the storage issue we have been having recently. Here is how it goes. …[idea overview]… But there are several outstanding problems with this approach. … [problem statement]… And here are the potential way to resolve them. …[solution]… I would like to get some feedback from the team and see if we can get aligned.”* + * **Potential issue**: potential stakeholders may not subscribe the stream where the message is posted and never participate in the discussion. * **Proposal**: Tag people/team whom you want to participate in the discussion or who you think will find the discussion valuable. Pay special attention to the way you tag them though, to not make the topic ‘exclusive’ to those people. On the other hand, avoid using @ all or @ everyone tag as that can be spammy and too broad. The message sits there for two days, some people reads the message, but do not participate. + * **Potential issue**: As Nala’s message did not receive any response or confirmation of being read, Nala starts wondering whether no one has read the message or the topic is not interesting to anyone. - * **Proposal**: ‘Ack’ the message by adding an emoji or leave comments. You don’t need to add emoji for every single on-going conversation. Rather, adding an emoji at the last message of the latest thread will do. + * **Proposal**: ‘Ack’ the message by adding an emoji or leave comments. You don’t need to add emoji for every single on-going conversation. Rather, adding an emoji at the last message of the latest thread will do. Eventually, Maukoo participates in the discussion. @@ -54,11 +56,13 @@ Then network expert Coriander participates as well. Another person Simba also contributes. -*“Oh, I remember that network issue. I wonder if we can do this. …[suggesting solution]…”* +*“Oh, I remember that network issue. I wonder if we can do this. …[suggesting solution]…”* + * **Potential issue**: Discussion is deviated from the original topic and the thread becomes harder to follow. * **Proposal**: Point out that the conversation is starting to diverge and try to bring the crowd back to the main topic. If the newly presented topic is worth a discussion, it can be done in a separate thread. The topic discussion is ongoing for several days. Nala has to take care of other matters so decides to leave the discussion open until they are done with the other matters. + * **Potential issue**: As the discussion gets long, it can be overwhelming for a new person to participate if they have to read 50 previous messages. * **Proposal**: Occasionally summarizing the discussion up to certain point can be helpful for everyone to quickly gather/refresh context. If it is an important discussion that better be documented, consider creating a Google doc so discussion points can be structured and archived. @@ -66,5 +70,6 @@ The topic discussion is ongoing for several days. Nala has to take care of other * **Proposal**: As someone who started the topic, it is your responsibility to make sure that the topic thread stays alive. If you cannot get back to this anytime soon, let the stakeholders know and share when you plan to pick up the discussion again so everyone can set the right expectation. Some time later, Nala asynchronously works with other engineers and align on the solution. + * **Potential issue**: From the Zulip topic point of view, the topic is never resolved and discussion is dropped in the middle. As a result, people who were not part of the offline discussion do not know the final state of the topic. * **Proposal**: Set the high level timeline on when the discussion needs to reach to the next stage (e.g. collect feedback in week 1, finalize solution in week 2, resolve the topic in week 3) and update the topic thread accordingly. Once the discussion is completed, share the outcome and resolve the topic thread. diff --git a/docs/practices/fast_builds.md b/docs/practices/fast_builds.md index 97339e03d02..e3ead617f8c 100644 --- a/docs/practices/fast_builds.md +++ b/docs/practices/fast_builds.md @@ -45,7 +45,7 @@ rustflags = ["-C", "link-arg=-fuse-ld=lld"] to `~/.cargo/config` is the most convenient approach. -`lld` itself can be installed with `sudo apt install lld` (or the equivalent in +`lld` itself can be installed with `sudo apt install lld` (or the equivalent in the distro/package manager of your choice). ## Prebuilt RocksDB @@ -59,8 +59,8 @@ To use a prebuilt RocksDB, set the `ROCKSDB_LIB_DIR` environment variable to a location containing `librocksdb.a`: ```console -$ export ROCKSDB_LIB_DIR=/usr/lib/x86_64-linux-gnu -$ cargo build -p neard +export ROCKSDB_LIB_DIR=/usr/lib/x86_64-linux-gnu +cargo build -p neard ``` Note, that the system must provide a recent version of the library which, @@ -83,6 +83,7 @@ export ROCKSDB_LIB_DIR ## Global Compilation Cache + By default, Rust compiles incrementally, with the incremental cache and intermediate outputs stored in the project-local `./target` directory. @@ -92,10 +93,10 @@ works by intercepting calls to `rustc` and will fetch the cached outputs from the global cache whenever possible. This tool can be set up as such: ```console -$ cargo install sccache -$ export RUSTC_WRAPPER="sccache" -$ export SCCACHE_CACHE_SIZE="30G" -$ cargo build -p neard +cargo install sccache +export RUSTC_WRAPPER="sccache" +export SCCACHE_CACHE_SIZE="30G" +cargo build -p neard ``` Refer to the [project’s README](https://github.com/mozilla/sccache) for further diff --git a/docs/practices/protocol_upgrade.md b/docs/practices/protocol_upgrade.md index 2eed6ecbfd3..3a3d052ebb9 100644 --- a/docs/practices/protocol_upgrade.md +++ b/docs/practices/protocol_upgrade.md @@ -25,29 +25,29 @@ and that it doesn't break other parts of the system. ### Protocol version voting and upgrade -When a new neard version, containing a new protocol version, is released, all node maintainers need +When a new neard version, containing a new protocol version, is released, all node maintainers need to upgrade their binary. That typically means stopping neard, downloading or compiling the new neard -binary and restarting neard. However the protocol version of the whole network is not immediately -bumped to the new protocol version. Instead a process called voting takes place and determines if and -when the protocol version upgrade will take place. +binary and restarting neard. However the protocol version of the whole network is not immediately +bumped to the new protocol version. Instead a process called voting takes place and determines if and +when the protocol version upgrade will take place. -Voting is a fully automated process in which all block producers across the network vote in support +Voting is a fully automated process in which all block producers across the network vote in support or against upgrading the protocol version. The voting happens in the last block every epoch. Upgraded -nodes will begin voting in favour of the new protocol version after a predetermined date. The voting -date is configured by the release owner [like this](https://github.com/near/nearcore/commit/9b0275de057a01f87c259580f93e58f746da75aa). -Once at least 80% of the stake votes in favour of the protocol change in the last block of epoch X, the -protocol version will be upgraded in the first block of epoch X+2. +nodes will begin voting in favour of the new protocol version after a predetermined date. The voting +date is configured by the release owner [like this](https://github.com/near/nearcore/commit/9b0275de057a01f87c259580f93e58f746da75aa). +Once at least 80% of the stake votes in favour of the protocol change in the last block of epoch X, the +protocol version will be upgraded in the first block of epoch X+2. -For mainnet releases, the release on github typically happens on a Monday or Tuesday, the voting -typically happens a week later and the protocol version upgrade happens 1-2 epochs after the voting. This +For mainnet releases, the release on github typically happens on a Monday or Tuesday, the voting +typically happens a week later and the protocol version upgrade happens 1-2 epochs after the voting. This gives the node maintainers enough time to upgrade their neard nodes. The node maintainers can upgrade their nodes at any time between the release and the voting but it is recommended to upgrade soon after the -release. This is to accommodate for any database migrations or miscellaneous delays. +release. This is to accommodate for any database migrations or miscellaneous delays. -Starting a neard node with protocol version voting in the future in a network that is already operating -at that protocol version is supported as well. This is useful in the scenario where there is a mainnet +Starting a neard node with protocol version voting in the future in a network that is already operating +at that protocol version is supported as well. This is useful in the scenario where there is a mainnet security release where mainnet has not yet voted or upgraded to the new version. That same binary with -protocol voting date in the future can be released in testnet even though it has already upgraded to +protocol voting date in the future can be released in testnet even though it has already upgraded to the new protocol version. ### Nightly Protocol features @@ -56,8 +56,8 @@ To make protocol upgrades more robust, we introduce the concept of a nightly protocol version together with the protocol feature flags to allow easy testing of the cutting-edge protocol changes without jeopardizing the stability of the codebase overall. The use of the nightly and nightly_protocol for new features -is mandatory while the use of dedicated rust features for new protocol features -is optional and only recommended when necessary. Adding rust features leads to +is mandatory while the use of dedicated rust features for new protocol features +is optional and only recommended when necessary. Adding rust features leads to conditional compilation which is generally not developer friendly. In `Cargo.toml` file of the crates we have in nearcore, we introduce rust compile-time features `nightly_protocol` and `nightly`: @@ -72,17 +72,16 @@ nightly = [ where `nightly_protocol` is a marker feature that indicates that we are on nightly protocol whereas `nightly` is a collection of new protocol features -which also implies `nightly_protocol`. +which also implies `nightly_protocol`. -When it is not necessary to use a rust feature for the new protocol feature +When it is not necessary to use a rust feature for the new protocol feature the Cargo.toml file will remain unchanged. -When it is necessary to use a rust feature for the new protocol feature, it +When it is necessary to use a rust feature for the new protocol feature, it can be added to the Cargo.toml, to the nightly features. For example, when we introduce EVM as a new protocol change, suppose the current protocol version is 40, then we would do the following change in Cargo.toml: - ```toml nightly_protocol = [] nightly = [ @@ -132,7 +131,6 @@ It is worth mentioning that there are two types of checks related to protocol fe protocol feature is enabled. This check is optional and can only be used for nightly features. - ### Testing Nightly protocol features allow us to enable the most bleeding-edge code in some @@ -167,4 +165,3 @@ A feature stabilization request must be approved by at least **two** Unless it is a security-related fix, a protocol feature cannot be included in any release until at least **one** week after its stabilization. This is to ensure that feature implementation and stabilization are not rushed. - diff --git a/docs/practices/rust.md b/docs/practices/rust.md index 856d012a20a..bcdc3ead97b 100644 --- a/docs/practices/rust.md +++ b/docs/practices/rust.md @@ -51,6 +51,7 @@ This is a thing in its category, do check it out: ## Language Mastery + * [Rust for Rustaceans](https://nostarch.com/rust-rustaceans) — the book to read after "The Book". * [Tokio docs](https://tokio.rs/tokio/tutorial) explain asynchronous programming diff --git a/docs/practices/security_vulnerabilities.md b/docs/practices/security_vulnerabilities.md index 3ea2b329b4a..b6f9b981fa9 100644 --- a/docs/practices/security_vulnerabilities.md +++ b/docs/practices/security_vulnerabilities.md @@ -1,5 +1,6 @@ ## Security Vulnerabilities +
The intended audience of the information presented here is developers working @@ -26,11 +27,12 @@ security-sensitive issues. using the following commands: ```console - $ git remote add nearcore-public git@github.com:near/nearcore - $ git remote add nearcore-private git@github.com:near/nearcore-private - $ git fetch nearcore-public - $ git push nearcore-private nearcore-public/master:master + git remote add nearcore-public git@github.com:near/nearcore + git remote add nearcore-private git@github.com:near/nearcore-private + git fetch nearcore-public + git push nearcore-private nearcore-public/master:master ``` + 2. All security-sensitive issues must be created on the private nearcore repository. You must also assign one of the `[P-S0, P-S1]` labels to the issue to indicate the severity of the issue. The two criteria to use to help diff --git a/docs/practices/style.md b/docs/practices/style.md index 86fb127df8a..8d4977686ee 100644 --- a/docs/practices/style.md +++ b/docs/practices/style.md @@ -20,7 +20,7 @@ into this document. Use `rustfmt` for minor code formatting decisions. This rule is enforced by CI **Rationale:** `rustfmt` style is almost always good enough, even if not always -perfect. The amount of bikeshedding saved by `rustfmt` far outweighs any +perfect. The amount of bike shedding saved by `rustfmt` far outweighs any imperfections. ## Idiomatic Rust @@ -37,6 +37,7 @@ When in doubt, ask question in the [Rust stream or during code review. **Rationale:** + - *Consistency*: there's usually only one idiomatic solution amidst many non-idiomatic ones. - *Predictability*: you can use the APIs without consulting documentation. @@ -79,7 +80,7 @@ this [issue](https://github.com/rust-lang/rust/issues/62586). Various generic APIs in Rust often return references to data (`&T`). When `T` is a small `Copy` type like `i32`, you end up with `&i32` while many API expect -`i32`, so dereference has to happen _somewhere_. Prefer dereferencing as early +`i32`, so dereference has to happen *somewhere*. Prefer dereferencing as early as possible, typically in a pattern: ```rust @@ -148,8 +149,8 @@ aren’t clear cut, it’s usually better to err on side of more imperative styl Lastly, anecdotally the methods (e.g. when used with `chain` or `flat_map`) may lead to faster code. This intuitively makes sense but it’s worth to keep in -mind that compilers are pretty good at optimising and in practice may generate -optimal code anyway. Furthermore, optimising code for readability may be more +mind that compilers are pretty good at optimizing and in practice may generate +optimal code anyway. Furthermore, optimizing code for readability may be more important (especially outside of hot path) than small performance gains. ### Prefer `to_string` to `format!("{}")` @@ -279,7 +280,7 @@ f % 7 If you’re confident the arithmetic operation cannot fail, `x.checked_[add|sub|mul|div](y).expect("explanation why the operation is safe")` is a great -alternative, as it neatly documents not just the infallibility, but also _why_ that is the case. +alternative, as it neatly documents not just the infallibility, but also *why* that is the case. This convention may be enforced by the `clippy::arithmetic_side_effects` and `clippy::integer_arithmetic` lints. @@ -289,7 +290,7 @@ factors, most notably the compilation flags used. The quick explanation is that computations may panic (cause side effects) if the result has overflowed, and when built with optimizations enabled, these computations will wrap-around instead. -For nearcore and neard we have opted to enable the panicking behaviour regardless of the +For nearcore and neard we have opted to enable the panicking behavior regardless of the optimization level. By doing it this we hope to prevent accidental stabilization of protocol mis-features that depend on incorrect handling of these overflows or similarly scary silent bugs. The downside to this approach is that any such arithmetic operation now may cause a node to crash, @@ -301,22 +302,22 @@ project overall. ## Standard Naming -* Use `-` rather than `_` in crate names and in corresponding folder names. -* Avoid single-letter variable names especially in long functions. Common `i`, +- Use `-` rather than `_` in crate names and in corresponding folder names. +- Avoid single-letter variable names especially in long functions. Common `i`, `j` etc. loop variables are somewhat of an exception but since Rust encourages use of iterators those cases aren’t that common anyway. -* Follow standard [Rust naming patterns](https://rust-lang.github.io/api-guidelines/naming.html) such as: - * Don’t use `get_` prefix for getter methods. A getter method is one which +- Follow standard [Rust naming patterns](https://rust-lang.github.io/api-guidelines/naming.html) such as: + - Don’t use `get_` prefix for getter methods. A getter method is one which returns (a reference to) a field of an object. - * Use `set_` prefix for setter methods. An exception are builder objects + - Use `set_` prefix for setter methods. An exception are builder objects which may use different a naming style. - * Use `into_` prefix for methods which consume `self` and `to_` prefix for + - Use `into_` prefix for methods which consume `self` and `to_` prefix for methods which don’t. -* Use `get_block_header` rather than `get_header` for methods which return +- Use `get_block_header` rather than `get_header` for methods which return a block header. -* Don’t use `_by_hash` suffix for methods which lookup chain objects (blocks, +- Don’t use `_by_hash` suffix for methods which lookup chain objects (blocks, chunks, block headers etc.) by their hash (i.e. their primary identifier). -* Use `_by_height` and similar suffixes for methods which lookup chain objects +- Use `_by_height` and similar suffixes for methods which lookup chain objects (blocks, chunks, block headers etc.) by their height or other property which is not their hash. @@ -324,6 +325,7 @@ project overall. ## Documentation + When writing documentation in `.md` files, wrap lines at approximately 80 columns. diff --git a/docs/practices/testing/coverage.md b/docs/practices/testing/coverage.md index 9e85a7f9fc5..80d40881ff7 100644 --- a/docs/practices/testing/coverage.md +++ b/docs/practices/testing/coverage.md @@ -19,6 +19,7 @@ artifact results come in. ## Artifact Results We also push artifacts, as a result of each CI run. You can access them here: + 1. Click "Details" on one of the CI actions run on your PR (literally any one of the actions is fine, you can also access CI actions runs on any CI) 2. Click "Summary" on the top left of the opening page @@ -29,6 +30,7 @@ We also push artifacts, as a result of each CI run. You can access them here: 5. Downloading it will give you a zip file with the interesting files. In there, you can find: + - Two `-diff` files, that contain code coverage for the diff of your PR, to easily see exactly which lines are covered and which are not - Two `-full` folders, that contain code coverage for the whole repository @@ -36,7 +38,6 @@ In there, you can find: tests, and one `integration-` variant, that contains all the tests we currently have - **To check that your PR is properly tested**, if you want better quality coverage than what codecov "requires," you can have a look at `unit-diff`, because we agreed that we want unit tests to be able to detect most bugs diff --git a/docs/practices/testing/test_utils.md b/docs/practices/testing/test_utils.md index aa203316305..c6ceacfbbea 100644 --- a/docs/practices/testing/test_utils.md +++ b/docs/practices/testing/test_utils.md @@ -97,7 +97,7 @@ To wait/handle a given event (as a lot of network code is running in an async fa pm.events.recv_util(|event| match event {...}).await; ``` -## End to End +## End to End ### chain, runtime, signer @@ -115,4 +115,3 @@ In chain/client/src/test_utils.rs ```rust let (block, client, view_client) = setup(MANY_FIELDS); ``` - diff --git a/docs/practices/when_to_use_private_repository.md b/docs/practices/when_to_use_private_repository.md index b4a84dc0691..cb3b80d0c22 100644 --- a/docs/practices/when_to_use_private_repository.md +++ b/docs/practices/when_to_use_private_repository.md @@ -1,5 +1,6 @@ # When to use private Github repository +
The intended audience of the information presented here is developers working @@ -10,22 +11,22 @@ Are you a security researcher? Please report security vulnerabilities to
-## TL;DR; +## TL;DR Limit usage of nearcore-private to sensitive issues that need to be kept confidential - ## Details -By nature, NEAR protocol ecosystem is community driven effort, welcoming everyone to make contribution; we believe in power of community and openness and bring a greater good. +By nature, NEAR protocol ecosystem is community driven effort, welcoming everyone to make contribution; we believe in power of community and openness and bring a greater good. In this regard, by default, contributors create an issue/PR in a public space, such as [nearcore](https://github.com/near/nearcore). -However, from time to time, if a sensitive issue is discovered (e.g. security vulnerability), it cannot be tracked publicly; we need to limit the blast radius and conceal the issue from malicious actors. -For this purpose, we maintain a private fork, [nearcore-private](https://github.com/near/nearcore-private), of public repository, nearcore, without any changes on top, and track such issues/PRs, in essence all sensitive development. +However, from time to time, if a sensitive issue is discovered (e.g. security vulnerability), it cannot be tracked publicly; we need to limit the blast radius and conceal the issue from malicious actors. +For this purpose, we maintain a private fork, [nearcore-private](https://github.com/near/nearcore-private), of public repository, nearcore, without any changes on top, and track such issues/PRs, in essence all sensitive development. Due to criticality of its contents, the private repository has limited accessibility and special attention is needed on deciding when to use the private repository. Before creating an issue under nearcore-private, ask the following questions yourself: + * Does it contain information that can be exploited by malicious actors? * Does it need special attention beyond what's provided from public repository due to its sensitivity? * Does confidentiality of the issue overrule our principle of public transparency? @@ -33,4 +34,5 @@ Before creating an issue under nearcore-private, ask the following questions you Unless an answer to one of these questions is 'YES', the issue may better reside within public repository. If you are unsure, consult private repo auditors (e.g. @akhi3030, @mm-near) to get their thoughts. ## Extra + To learn more about the process for creating an issue/PR on nearcore-private, please visit [link](https://github.com/near/nearcore/blob/c308df157bf64a528033b618b4f444d3b9c73f94/docs/practices/security_vulnerabilities.md). diff --git a/docs/practices/workflows/README.md b/docs/practices/workflows/README.md index 6b771fcea8b..db49c64c997 100644 --- a/docs/practices/workflows/README.md +++ b/docs/practices/workflows/README.md @@ -1,4 +1,4 @@ # Workflows This chapter documents various ways you can run `neard` during development: -running a local net, joining a test net, doing benchmarking and load testing. +running a local net, joining a test net, doing benchmarking and load testing. diff --git a/docs/practices/workflows/deploy_a_contract.md b/docs/practices/workflows/deploy_a_contract.md index a1128792ed0..7dc1fd4430d 100644 --- a/docs/practices/workflows/deploy_a_contract.md +++ b/docs/practices/workflows/deploy_a_contract.md @@ -1,3 +1,4 @@ + # Deploy a Contract In this chapter, we'll learn how to build, deploy, and call a minimal smart @@ -10,9 +11,9 @@ deploy a contract. You might want to re-read [how to run a node](./run_a_node.md to understand what's going on here: ```console -$ cargo run --profile dev-release -p neard -- init -$ cargo run --profile dev-release -p neard -- run -$ NEAR_ENV=local near create-account alice.test.near --masterAccount test.near +cargo run --profile dev-release -p neard -- init +cargo run --profile dev-release -p neard -- run +NEAR_ENV=local near create-account alice.test.near --masterAccount test.near ``` As a sanity check, querying the state of `alice.test.near` account should work: @@ -80,13 +81,13 @@ Before we start doing that, some amount of setup code is required. Let's start with creating a new crate: ```console -$ cargo new hello-near --lib +cargo new hello-near --lib ``` To compile to wasm, we also need to add a relevant rustup toolchain: ```console -$ rustup toolchain add wasm32-unknown-unknown +rustup toolchain add wasm32-unknown-unknown ``` Then, we need to tell Cargo that the final artifact we want to get is a @@ -239,7 +240,6 @@ Done deploying to alice.test.near And, finally, let's call our contract: - ```console $ NEAR_ENV=local $near call alice.test.near hello --accountId alice.test.near Scheduling a call: alice.test.near.hello() @@ -256,7 +256,6 @@ we are calling, the second time to determine who calls the contract. That is, the second account is the one that spends tokens. In the following example `bob` spends NEAR to call the contact deployed to the `alice` account: - ```console $ NEAR_ENV=local $near call alice.test.near hello --accountId bob.test.near Scheduling a call: alice.test.near.hello() diff --git a/docs/practices/workflows/gas_estimations.md b/docs/practices/workflows/gas_estimations.md index ec2228a67c2..973ed7aa520 100644 --- a/docs/practices/workflows/gas_estimations.md +++ b/docs/practices/workflows/gas_estimations.md @@ -62,6 +62,7 @@ Note how the output looks a bit different now. The `i`, `r` and `w` values show instruction count, read IO bytes, and write IO bytes respectively. The IO byte count is known to be inaccurate. + ``` + /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/qemu-x86_64 -plugin file=/host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/libcounter.so -cpu Haswell-v4 /host/nearcore/target/release/runtime-params-estimator --home /.near --accounts-num 20000 --iters 3 --warmup-iters 1 --metric icount --costs=ActionReceiptCreation,ActionTransfer,ActionCreateAccount,ActionFunctionCallBase --skip-build-test-contract --additional-accounts-num 0 --in-memory-db ActionReceiptCreation 214_581_685_500 gas [ 1716653.48i 0.00r 0.00w] (computed in 6.11s) diff --git a/docs/practices/workflows/io_trace.md b/docs/practices/workflows/io_trace.md index 6fe55df8aae..ef5a4aa82fd 100644 --- a/docs/practices/workflows/io_trace.md +++ b/docs/practices/workflows/io_trace.md @@ -6,14 +6,14 @@ IO traces can be used to identify slow receipts and to understand why they are slow. Or to detect general inefficiencies in how we use the DB. The results give you counts of DB requests and some useful statistics such as -trie node cache hits. It will NOT give you time measurements, use Graphana to +trie node cache hits. It will NOT give you time measurements, use Grafana to observe those. The main use cases in the past were to estimate the performance of new storage features (prefetcher, flat state) and to find out why specific contracts produce slow receipts. -## Setup +## Setup When compiling neard (or the parameter estimator) with `feature=io_trace` it instruments the binary code with fine-grained database operations tracking. @@ -62,6 +62,7 @@ Once you have collected an IO trace, you can inspect its content manually, or use existing tools to extract statistics. Let's start with the manual approach. ### Simple example trace: Estimator + An estimator trace typically starts something like this: ``` @@ -99,6 +100,7 @@ cache-missed all 7 of the DB requests it performed to apply this empty chunk. ### Example trace: Full mainnet node + Next let's look at an excerpt of an IO trace from a real node on mainnet. ``` @@ -173,7 +175,7 @@ but there are DB requests listed further below that belong to these levels but not to `apply`. Inside `apply`, we see 3 transactions being converted to receipts as part of -this chunk, and one already existing action receipt getting processed. +this chunk, and one already existing action receipt getting processed. Cache hit statistics for each level are also displayed. For example, the first transaction has 57 read requests and all of them hit in the shard cache. For @@ -236,7 +238,7 @@ Afterwards, a value with 16 bytes (a `u128`) is fetched from the trie state. To serve this, it required reading 30 trie nodes, 19 of them were cached in the accounting cache and were not charged the full gas cost. And the remaining 11 missed the accounting cache but they hit the shard cache. Nothing needed to be -fetched from DB because the Sweatcoin specific prefetcher has already loaded +fetched from DB because the SweatCoin specific prefetcher has already loaded everything into the shard cache. *Note: We see trie node requests despite flat state being used. This is because @@ -257,7 +259,6 @@ flat state to do some correctness checks.* So that is how to read these traces and dig deep. But maybe you want aggregated statistics instead? Then please continue reading. - ## Evaluating an IO trace When you collect an IO trace over an hour of mainnet traffic, it can quickly be @@ -279,7 +280,6 @@ All commands aggregate the information of a trace. Either globally, per chunk, or per receipt. For example, below is the output that gives a list of RocksDB columns that were accessed and how many times, aggregated by chunk. - ```bash cargo run --profile dev-release -p runtime-params-estimator -- \ replay ./path/to/my.io_trace chunk-db-stats @@ -320,4 +320,4 @@ not be assigned to specific chunks. The way we currently count write operations (SET, UPDATE_RC) they are never assigned to a specific chunk and instead only show up in the top-level list. Clearly, there is some room for improvement here. So far we simply haven't worried about RocksDB write performance so the tooling -to debug write performance is naturally lacking. \ No newline at end of file +to debug write performance is naturally lacking. diff --git a/docs/practices/workflows/localnet_on_many_machines.md b/docs/practices/workflows/localnet_on_many_machines.md index c351c0e9e30..ed2fbc3abd0 100644 --- a/docs/practices/workflows/localnet_on_many_machines.md +++ b/docs/practices/workflows/localnet_on_many_machines.md @@ -7,8 +7,8 @@ Quick instructions on how to run a localnet on 2 separate machines. * Machine1: "pc" - 192.168.0.1 * Machine2: "laptop" - 192.168.0.2 - Run on both machines (make sure that they are using the same version of the code): + ``` cargo build -p neard ``` @@ -28,40 +28,42 @@ rsync -r ~/.near/localnet_multi/node1 192.168.0.2:~/.near/localnet_multi/node1 ``` Now open the config.json file on both machines (node0/config.json on machine1 and node1/config.json on machine2) and: + * for rpc->addr and network->addr: * Change the address from 127.0.0.1 to 0.0.0.0 (this means that the port will be accessible from other computers) - * Remember the port numbers (they are generated randomly). + * Remember the port numbers (they are generated randomly). * Also write down the node0's node_key (it is probably: "ed25519:7PGseFbWxvYVgZ89K1uTJKYoKetWs7BJtbyXDzfbAcqX") ## Running On machine1: + ``` ./target/debug/neard --home ~/.near/localnet_multi/node0 run ``` On machine2: + ``` ./target/debug/neard --home ~/.near/localnet_multi/node1 run --boot-nodes ed25519:7PGseFbWxvYVgZ89K1uTJKYoKetWs7BJtbyXDzfbAcqX@192.168.0.1:37665 ``` -The boot node address should be the IP of the machine1 + the network addr port **from the node0/config.json** +The boot node address should be the IP of the machine1 + the network addr port **from the node0/config.json** And if everything goes well, the nodes should communicate and start producing blocks. ## Troubleshooting -The debug mode is enabled by default, so you should be able to see what's going on by going to ``http://machine1:RPC_ADDR_PORT/debug`` - +The debug mode is enabled by default, so you should be able to see what's going on by going to ``http://machine1:RPC_ADDR_PORT/debug`` ### If node keeps saying "waiting for peers" + See if you can see the machine1's debug page from machine2. (if not - there might be a firewall blocking the connection). Make sure that you set the right ports (it should use node0's NETWORK port) and that you set the ip add there to 0.0.0.0 - ### Resetting the state + Simply stop both nodes, and remove the ``data`` subdirectory (~/.near/localnet_multi/node0/data and ~/.near/localnet_multi/node1/data). Then after restart, the nodes will start the blockchain from scratch. - diff --git a/docs/practices/workflows/otel_traces.md b/docs/practices/workflows/otel_traces.md index 6e6f2a34d88..0ad0fe52806 100644 --- a/docs/practices/workflows/otel_traces.md +++ b/docs/practices/workflows/otel_traces.md @@ -1,3 +1,4 @@ + # Working with OpenTelemetry Traces `neard` is instrumented in a few different ways. From the code perspective we have two major ways @@ -45,10 +46,8 @@ exclusively on logs only increases noise for the other developers and makes it e harder to extract signal in the future. Keep this trade off in mind. [Tempo]: https://grafana.com/oss/tempo/ -[Loki]: https://grafana.com/oss/loki/ [Jaeger]: https://www.jaegertracing.io/ - ### Spans We have a [style guide section on the use of Spans](../style.md#spans), please make yourself @@ -77,8 +76,8 @@ only info-level spans doesn’t give any useful tracing information in Grafana. lets us easily annotate every implementation of `actix::Actor::handle()`. This macro sets the following span attributes: - * `actor` to the name of the struct that implements actix::Actor - * `handler` to the name of the message struct + * `actor` to the name of the struct that implements actix::Actor + * `handler` to the name of the message struct And it lets you provide more span attributes. In the example, ClientActor specifies `msg_type`, which in all cases is identical to `handler`. diff --git a/docs/practices/workflows/profiling.md b/docs/practices/workflows/profiling.md index 97cacbd1a8c..374f2dda7b8 100755 --- a/docs/practices/workflows/profiling.md +++ b/docs/practices/workflows/profiling.md @@ -1,3 +1,4 @@ + # Profiling neard ## Sampling performance profiling @@ -12,8 +13,8 @@ Linux's `perf` has been a tool of choice in most cases, although tools like Inte used too. In order to use either, first prepare your system: ```command -$ sudo sysctl kernel.perf_event_paranoid=0 -$ sudo sysctl kernel.kptr_restrict=0 +sudo sysctl kernel.perf_event_paranoid=0 +sudo sysctl kernel.kptr_restrict=0 ```
@@ -44,7 +45,7 @@ Profiler supports `perf` and many other different data formats, for `perf` in pa conversion step is necessary: ```command -$ perf script -F +pid > mylittleprofile.script +perf script -F +pid > mylittleprofile.script ``` Then, load this `mylittleprofile.script` file with the profiler. @@ -58,13 +59,13 @@ ability to build a profiling-tuned build of `neard`, you can use higher quality collection. ```command -$ cargo build --release --config .cargo/config.profiling.toml -p neard +cargo build --release --config .cargo/config.profiling.toml -p neard ``` Then, replace the `--call-graph dwarf` with `--call-graph fp`: ```command -$ perf record -e cpu-clock -F1000 -g --call-graph fp,65528 YOUR_COMMAND_HERE +perf record -e cpu-clock -F1000 -g --call-graph fp,65528 YOUR_COMMAND_HERE ``` ### Profiling with hardware counters @@ -98,15 +99,15 @@ Once everything is set up, though, the following command can gather some interes for you. ```command -$ perf record -e cycles:u -b -g --call-graph fp,65528 YOUR_COMMAND_HERE +perf record -e cycles:u -b -g --call-graph fp,65528 YOUR_COMMAND_HERE ``` Analyzing this data is, unfortunately, not as easy as chucking it away to Firefox Profiler. I'm not aware of any other ways to inspect the data other than using `perf report`: ```command -$ perf report -g --branch-history -$ perf report -g --branch-stack +perf report -g --branch-history +perf report -g --branch-stack ``` You may also be able to gather some interesting results if you use `--call-graph lbr` and the @@ -119,7 +120,7 @@ Although Rust makes it pretty hard to introduce memory problems, it is still pos memory or to inadvertently retain too much of it. Unfortunately, “just” throwing a random profiler at neard does not work for many reasons. Valgrind -for example is introducing enough slowdown to significantly alter the behaviour of the run, not to +for example is introducing enough slowdown to significantly alter the behavior of the run, not to mention that to run it successfully and without crashing it will be necessary to comment out `neard`’s use of `jemalloc` for yet another substantial slowdown. @@ -131,24 +132,24 @@ First, checkout and build the profiler (you will need to have nodejs `yarn` thin well): ```command -$ git clone git@github.com:koute/bytehound.git -$ cargo build --release -p bytehound-preload -$ cargo build --release -p bytehound-cli +git clone git@github.com:koute/bytehound.git +cargo build --release -p bytehound-preload +cargo build --release -p bytehound-cli ``` You will also need a build of your `neard`, once you have that, give it some ambient capabilities necessary for profiling: ```command -$ sudo sysctl kernel.perf_event_paranoid=0 -$ sudo setcap 'CAP_SYS_ADMIN+ep' /path/to/neard -$ sudo setcap 'CAP_SYS_ADMIN+ep' /path/to/libbytehound.so +sudo sysctl kernel.perf_event_paranoid=0 +sudo setcap 'CAP_SYS_ADMIN+ep' /path/to/neard +sudo setcap 'CAP_SYS_ADMIN+ep' /path/to/libbytehound.so ``` And finally run the program with the profiler enabled (in this case `neard run` command is used): ```command -$ /lib64/ld-linux-x86-64.so.2 --preload /path/to/libbytehound.so /path/to/neard run +/lib64/ld-linux-x86-64.so.2 --preload /path/to/libbytehound.so /path/to/neard run ``` ### Viewing the profile @@ -203,7 +204,6 @@ We don't know what exactly it is about neard that leads to it crashing under the as it does. I have seen valgrind reporting that we have libraries that are deallocating with a wrong size class, so that might be the reason? Do definitely look into this if you have time. - ## What to profile? This section provides some ideas on programs you could consider profiling if you are not sure where diff --git a/docs/practices/workflows/run_a_node.md b/docs/practices/workflows/run_a_node.md index 0b39e5cb504..803bafc0566 100644 --- a/docs/practices/workflows/run_a_node.md +++ b/docs/practices/workflows/run_a_node.md @@ -9,7 +9,7 @@ relatively little attention to the various shortcuts we have. Start with the following command: ```console -$ cargo run --profile dev-release -p neard -- --help +cargo run --profile dev-release -p neard -- --help ``` This command builds `neard` and asks it to show `--help`. Building `neard` takes @@ -142,7 +142,7 @@ $ cat ~/.near/genesis.json | jq '.validators' Now, if we ```console -$ cat ~/.near/validator_key.json +cat ~/.near/validator_key.json ``` we'll see @@ -191,7 +191,7 @@ is rejected. ## Running the Network Finally, - + ```console $ cargo run --profile dev-release -p neard -- run INFO neard: version="trunk" build="1.1.0-3091-ga8964d200-modified" latest_protocol=57 @@ -428,7 +428,7 @@ http://localhost:9001/transactions/BBPndo6gR4X8pzoDK7UQfoUXp5J8WDxkf8Sq75tK5FFT to do multiple commands to avoid repetition: ```console -$ export NEAR_ENV=local +export NEAR_ENV=local ``` NEAR CLI printouts are not always the most useful or accurate, but this seems to diff --git a/docs/test_networks/mainnet_spoon.md b/docs/test_networks/mainnet_spoon.md index 3d9c2d4c280..67ed2a70e8e 100644 --- a/docs/test_networks/mainnet_spoon.md +++ b/docs/test_networks/mainnet_spoon.md @@ -24,7 +24,7 @@ you have your node's home directory set up, run the following `state-viewer` command to generate a dump of the chain's state: ```shell -$ neard --home $NEAR_HOME_DIRECTORY view-state dump-state --stream +neard --home $NEAR_HOME_DIRECTORY view-state dump-state --stream ``` This command will take a while (possibly many hours) to run. But at the @@ -48,7 +48,7 @@ chain. Suppose that we want our chain to have two validators, where we'll be storing intermediate files during this process: ```shell -$ mkdir ~/test-chain-scratch +mkdir ~/test-chain-scratch ``` then using your favorite editor, lay out the validators you want in @@ -75,8 +75,8 @@ These validator keys should be keys you've already generated. So for the rest of this document, we'll assume you've run: ```shell -$ neard --home ~/near-test-chain/validator0 init --account-id validator0.near -$ neard --home ~/near-test-chain/validator1 init --account-id validator1.near +neard --home ~/near-test-chain/validator0 init --account-id validator0.near +neard --home ~/near-test-chain/validator1 init --account-id validator1.near ``` This is also a good time to think about what extra accounts you might @@ -146,10 +146,10 @@ copy the records and genesis files generated in the previous step to each of these: ```shell -$ cp ~/near-test-chain/records.json ~/near-test-chain/validator0 -$ cp ~/near-test-chain/genesis.json ~/near-test-chain/validator0 -$ cp ~/near-test-chain/records.json ~/near-test-chain/validator1 -$ cp ~/near-test-chain/genesis.json ~/near-test-chain/validator1 +cp ~/near-test-chain/records.json ~/near-test-chain/validator0 +cp ~/near-test-chain/genesis.json ~/near-test-chain/validator0 +cp ~/near-test-chain/records.json ~/near-test-chain/validator1 +cp ~/near-test-chain/genesis.json ~/near-test-chain/validator1 ``` Now we'll need to make a few config changes to each of @@ -212,12 +212,12 @@ After making these changes, you can try running one neard process for each of your validators: ```shell -$ neard --home ~/near-test-chain/validator0 run -$ neard --home ~/near-test-chain/validator1 run +neard --home ~/near-test-chain/validator0 run +neard --home ~/near-test-chain/validator1 run ``` Now these nodes will begin by taking the records laid out in `records.json` and turning them into a genesis state. At the time of this writing, using the latest nearcore version from the master branch, this will take a couple hours. But your validators should -begin producing blocks after that's done. \ No newline at end of file +begin producing blocks after that's done. From c675c49a64b5355d96cdb61de98fdda7264fd027 Mon Sep 17 00:00:00 2001 From: Shreyan Gupta Date: Fri, 24 Jan 2025 11:15:46 +0530 Subject: [PATCH 2/3] fix --- cspell.json | 13 +++- debug_scripts/estimate_epoch_start_time.py | 71 ++++++++++--------- debug_scripts/request_chain_info.py | 38 +++++----- debug_scripts/send_validator_logs.py | 37 +++++----- .../tests/send_validator_logs_test.py | 13 ++-- 5 files changed, 95 insertions(+), 77 deletions(-) diff --git a/cspell.json b/cspell.json index f16feb54fcb..204502f2443 100644 --- a/cspell.json +++ b/cspell.json @@ -2,12 +2,13 @@ "version": "0.2", "ignorePaths": [ "*.ros", + "*.toml", "benchmarks/continuous/db/tool/orm/migrations/**", "chain/rosetta-rpc/src/adapters/snapshots/**", "core/primitives/res/epoch_configs/**.json", "cspell.json", "debug_scripts/Pipfile", - "docs/images/**", + "docs/images/**" ], "dictionaryDefinitions": [], "dictionaries": [], @@ -85,8 +86,10 @@ "Ggas", "gprusak", "hashlib", + "icount", "idempotently", "illia", + "indicatif", "insta", "Interruptible", "itertools", @@ -101,9 +104,12 @@ "kickout", "Kickouts", "kkuuue", + "libcore", "libfuzzer", "librocksdb", "lightclient", + "lightclients", + "loadtesting", "localnet", "Logunov", "Mbps", @@ -124,6 +130,7 @@ "mocknet", "multiexp", "multinode", + "multishard", "multizip", "nagisa", "nanos", @@ -136,6 +143,8 @@ "nearone", "nearprotocol", "nearup", + "nearvm", + "newtype", "nextest", "nikurt", "NOFILE", @@ -209,6 +218,7 @@ "Seedable", "serde", "Shreyan", + "Shutdownable", "smallvec", "spammy", "splitdb", @@ -246,6 +256,7 @@ "uids", "uncompiled", "uncongested", + "unflushed", "unittests", "unlabel", "unorphaned", diff --git a/debug_scripts/estimate_epoch_start_time.py b/debug_scripts/estimate_epoch_start_time.py index d70e56632de..d59e21e090e 100644 --- a/debug_scripts/estimate_epoch_start_time.py +++ b/debug_scripts/estimate_epoch_start_time.py @@ -14,9 +14,11 @@ def get_block(url, block_hash): "method": "block", } - payload["params"] = ( - {"block_id": block_hash} if block_hash is not None else {"finality": "final"} - ) + payload["params"] = ({ + "block_id": block_hash + } if block_hash is not None else { + "finality": "final" + }) response = requests.post(url, json=payload) return response.json()["result"]["header"] @@ -32,9 +34,10 @@ def format_time(seconds): # Function to fetch epoch lengths for the past n epochs and calculate the weighted average using exponential decay -def get_exponential_weighted_epoch_lengths( - url, starting_block_hash, num_epochs, decay_rate=0.1 -): +def get_exponential_weighted_epoch_lengths(url, + starting_block_hash, + num_epochs, + decay_rate=0.1): epoch_lengths = [] current_hash = starting_block_hash @@ -88,13 +91,11 @@ def valid_timezone(timezone_str): # Function to approximate future epoch start dates -def predict_future_epochs( - starting_epoch_timestamp, avg_epoch_length, num_future_epochs, target_timezone -): +def predict_future_epochs(starting_epoch_timestamp, avg_epoch_length, + num_future_epochs, target_timezone): future_epochs = [] current_timestamp = ns_to_seconds( - starting_epoch_timestamp - ) # Convert from nanoseconds to seconds + starting_epoch_timestamp) # Convert from nanoseconds to seconds for i in range(1, num_future_epochs + 1): # Add the average epoch length for each future epoch @@ -102,7 +103,8 @@ def predict_future_epochs( future_epochs.append(future_timestamp) # Convert timestamp to datetime in target timezone - future_datetime = datetime.fromtimestamp(future_timestamp, target_timezone) + future_datetime = datetime.fromtimestamp(future_timestamp, + target_timezone) # Format date future_date = future_datetime.strftime("%Y-%m-%d %H:%M:%S %Z%z %A") @@ -128,8 +130,7 @@ def find_best_voting_hour(voting_date_str, future_epochs): for hour in range(24): # Construct datetime for each hour of the voting date voting_datetime = datetime.strptime( - f"{voting_date_str} {hour:02d}:00:00", "%Y-%m-%d %H:%M:%S" - ) + f"{voting_date_str} {hour:02d}:00:00", "%Y-%m-%d %H:%M:%S") voting_datetime = pytz.utc.localize(voting_datetime) voting_timestamp = voting_datetime.timestamp() @@ -146,8 +147,10 @@ def find_best_voting_hour(voting_date_str, future_epochs): ) break - protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - 1] - protocol_upgrade_datetime = datetime.fromtimestamp(protocol_upgrade_timestamp) + protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number + - 1] + protocol_upgrade_datetime = datetime.fromtimestamp( + protocol_upgrade_timestamp) upgrade_hour_utc = protocol_upgrade_datetime.hour if WORKING_HOURS_START <= upgrade_hour_utc < WORKING_HOURS_END: @@ -179,7 +182,8 @@ def find_protocol_upgrade_time(voting_date, future_epochs, target_timezone): voting_datetime = target_timezone.localize(voting_date) # Find the epoch T in which the voting date falls - epoch_T = find_epoch_for_timestamp(future_epochs, voting_datetime.timestamp()) + epoch_T = find_epoch_for_timestamp(future_epochs, + voting_datetime.timestamp()) if epoch_T <= 0: print("Error: Voting date is before the first predicted epoch.") return @@ -191,13 +195,12 @@ def find_protocol_upgrade_time(voting_date, future_epochs, target_timezone): "Not enough future epochs predicted to determine the protocol upgrade time." ) return - protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - 1] + protocol_upgrade_timestamp = future_epochs[protocol_upgrade_epoch_number - + 1] protocol_upgrade_datetime = datetime.fromtimestamp( - protocol_upgrade_timestamp, tz=target_timezone - ) + protocol_upgrade_timestamp, tz=target_timezone) protocol_upgrade_formatted = protocol_upgrade_datetime.strftime( - "%Y-%m-%d %H:%M:%S %Z%z %A" - ) + "%Y-%m-%d %H:%M:%S %Z%z %A") print(f"\nVoting date falls into epoch {epoch_T}.") print( f"Protocol upgrade will happen at the start of epoch {protocol_upgrade_epoch_number}: {protocol_upgrade_formatted}" @@ -209,16 +212,14 @@ def main(args): latest_block = get_block(args.url, None) next_epoch_id = latest_block["next_epoch_id"] current_epoch_first_block = get_block(args.url, next_epoch_id) - current_timestamp = int( - current_epoch_first_block["timestamp"] - ) # Current epoch start timestamp in nanoseconds + current_timestamp = int(current_epoch_first_block["timestamp"] + ) # Current epoch start timestamp in nanoseconds # Get epoch lengths and the exponential weighted average epoch_lengths, exponential_weighted_average_epoch_length = ( - get_exponential_weighted_epoch_lengths( - args.url, next_epoch_id, args.num_past_epochs, args.decay_rate - ) - ) + get_exponential_weighted_epoch_lengths(args.url, next_epoch_id, + args.num_past_epochs, + args.decay_rate)) # Predict future epoch start dates future_epochs = predict_future_epochs( @@ -229,7 +230,8 @@ def main(args): ) if args.voting_date: - find_protocol_upgrade_time(args.voting_date, future_epochs, args.timezone) + find_protocol_upgrade_time(args.voting_date, future_epochs, + args.timezone) elif args.voting_date_day: find_best_voting_hour(args.voting_date_day, future_epochs) @@ -247,8 +249,7 @@ def __call__(self, parser, namespace, values, option_string=None): # Set up command-line argument parsing if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Approximate future epoch start dates for NEAR Protocol." - ) + description="Approximate future epoch start dates for NEAR Protocol.") # Create a mutually exclusive group for chain_id and url group = parser.add_mutually_exclusive_group(required=False) group.add_argument("--url", help="The RPC URL to query.") @@ -256,7 +257,8 @@ def __call__(self, parser, namespace, values, option_string=None): "--chain_id", choices=["mainnet", "testnet"], action=SetURLFromChainID, - help="The chain ID (either 'mainnet' or 'testnet'). Sets the corresponding URL.", + help= + "The chain ID (either 'mainnet' or 'testnet'). Sets the corresponding URL.", ) parser.add_argument( @@ -292,7 +294,8 @@ def __call__(self, parser, namespace, values, option_string=None): ) voting_group.add_argument( "--voting_date_day", - help="Voting date (day) in 'YYYY-MM-DD' format to find voting hours resulting in upgrade during working hours.", + help= + "Voting date (day) in 'YYYY-MM-DD' format to find voting hours resulting in upgrade during working hours.", ) args = parser.parse_args() diff --git a/debug_scripts/request_chain_info.py b/debug_scripts/request_chain_info.py index f51a8b27022..0b1de84a2e6 100755 --- a/debug_scripts/request_chain_info.py +++ b/debug_scripts/request_chain_info.py @@ -6,30 +6,30 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( - description="This is a script to request for blockchain info" - ) - parser.add_argument( - "--chain", choices=["mainnet", "testnet", "betanet"], required=True - ) - parser.add_argument( - "--archive", action="store_true", help="whether to request from archival nodes" - ) - parser.add_argument( - "--method", choices=["block", "chunk"], required=True, help="type of request" - ) - parser.add_argument( - "--block_id", type=str, help="block id, can be either block height or hash" - ) + description="This is a script to request for blockchain info") + parser.add_argument("--chain", + choices=["mainnet", "testnet", "betanet"], + required=True) + parser.add_argument("--archive", + action="store_true", + help="whether to request from archival nodes") + parser.add_argument("--method", + choices=["block", "chunk"], + required=True, + help="type of request") + parser.add_argument("--block_id", + type=str, + help="block id, can be either block height or hash") parser.add_argument("--shard_id", type=int, help="shard id for the chunk") parser.add_argument("--chunk_id", type=str, help="chunk hash") - parser.add_argument( - "--result_key", type=str, nargs="*", help="filter results by these keys" - ) + parser.add_argument("--result_key", + type=str, + nargs="*", + help="filter results by these keys") args = parser.parse_args() url = "https://{}.{}.near.org".format( - "archival-rpc" if args.archive else "rpc", args.chain - ) + "archival-rpc" if args.archive else "rpc", args.chain) def get_block_id(block_id): if block_id.isnumeric(): diff --git a/debug_scripts/send_validator_logs.py b/debug_scripts/send_validator_logs.py index 65890f4e2da..b75eb5c3519 100644 --- a/debug_scripts/send_validator_logs.py +++ b/debug_scripts/send_validator_logs.py @@ -8,9 +8,8 @@ import urllib.parse -def filter_log_file( - log_file: str, start_time: datetime.datetime, end_time: datetime.datetime -) -> list: +def filter_log_file(log_file: str, start_time: datetime.datetime, + end_time: datetime.datetime) -> list: """ Filter log file for a time range. start_time: datetime.datetime @@ -29,9 +28,9 @@ def filter_log_file( for line in f: # [0m and [2m are ANSI shell color codes. Removing them to parse dates. split_lines = line.split("[0m", 1)[0].replace("\x1b[2m", "") - dt = datetime.datetime.strptime(split_lines[:-5], "%b %d %H:%M:%S").replace( - year=datetime.datetime.now().year - ) + dt = datetime.datetime.strptime( + split_lines[:-5], + "%b %d %H:%M:%S").replace(year=datetime.datetime.now().year) if start_time <= dt <= end_time: filtered_logs.append(line) return filtered_logs @@ -59,9 +58,8 @@ def upload_to_s3(file_lines: list, account: str) -> str: ) s3 = boto3.resource("s3") - s3.Bucket(BUCKET).upload_fileobj( - io.BytesIO(gzipped_content), f"logs/{s3_destination}" - ) + s3.Bucket(BUCKET).upload_fileobj(io.BytesIO(gzipped_content), + f"logs/{s3_destination}") s3_link = ( f"https://{BUCKET}.s3.amazonaws.com/logs/{urllib.parse.quote(s3_destination)}" ) @@ -72,10 +70,14 @@ def upload_to_s3(file_lines: list, account: str) -> str: if __name__ == "__main__": parser = argparse.ArgumentParser(description="Send logs to near.") - parser.add_argument( - "--log_file", type=str, help="Absolute path to log file.", required=True - ) - parser.add_argument("--account", type=str, help="Near account id.", required=True) + parser.add_argument("--log_file", + type=str, + help="Absolute path to log file.", + required=True) + parser.add_argument("--account", + type=str, + help="Near account id.", + required=True) parser.add_argument( "--last_seconds", type=int, @@ -86,9 +88,10 @@ def upload_to_s3(file_lines: list, account: str) -> str: log_file_path = args.log_file end_timestamp = datetime.datetime.utcnow() - start_timestamp = end_timestamp - datetime.timedelta(seconds=args.last_seconds) + start_timestamp = end_timestamp - datetime.timedelta( + seconds=args.last_seconds) - filtered_log_lines = filter_log_file( - log_file=args.log_file, start_time=start_timestamp, end_time=end_timestamp - ) + filtered_log_lines = filter_log_file(log_file=args.log_file, + start_time=start_timestamp, + end_time=end_timestamp) upload_to_s3(file_lines=filtered_log_lines, account=args.account) diff --git a/debug_scripts/tests/send_validator_logs_test.py b/debug_scripts/tests/send_validator_logs_test.py index 07dbd4c751e..9cb1c91dbff 100644 --- a/debug_scripts/tests/send_validator_logs_test.py +++ b/debug_scripts/tests/send_validator_logs_test.py @@ -10,13 +10,14 @@ class test_validator_log_filtering(unittest.TestCase): def test_time_filtering(self): start_time = datetime.datetime(2022, 4, 4, 23, 42, 0, 0) end_time = datetime.datetime(2022, 4, 4, 23, 49, 0, 0) - output_file_obj = filter_log_file( - "./tests/data/node0.logs", start_time=start_time, end_time=end_time - ) + output_file_obj = filter_log_file("./tests/data/node0.logs", + start_time=start_time, + end_time=end_time) self.assertIsInstance( - output_file_obj, list, "Parsed file object should be of type io.BytesIO" - ) - self.assertEqual(len(output_file_obj), 48, "Filtered log should have 48 lines") + output_file_obj, list, + "Parsed file object should be of type io.BytesIO") + self.assertEqual(len(output_file_obj), 48, + "Filtered log should have 48 lines") if __name__ == "__main__": From c632281e642000bc4f023b8d8e8ea0594bd1dbea Mon Sep 17 00:00:00 2001 From: Shreyan Gupta Date: Fri, 24 Jan 2025 13:59:13 +0530 Subject: [PATCH 3/3] comments --- cspell.json | 37 +++++++++++++++++++++-- docs/architecture/storage/trie_storage.md | 2 +- docs/practices/workflows/io_trace.md | 3 +- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/cspell.json b/cspell.json index 204502f2443..99daff72c0e 100644 --- a/cspell.json +++ b/cspell.json @@ -8,7 +8,9 @@ "core/primitives/res/epoch_configs/**.json", "cspell.json", "debug_scripts/Pipfile", - "docs/images/**" + "docs/images/**", + "pytest/requirements.txt", + "pytest/tests/sandbox/patch_state.py" ], "dictionaryDefinitions": [], "dictionaries": [], @@ -16,6 +18,7 @@ "acks", "actix", "addrs", + "adversenet", "AFAIU", "akhi", "Aleksandr", @@ -30,10 +33,10 @@ "benchmarknet", "betanet", "bijective", + "bintools", "bitmask", "bitvec", "BLOCKLIST", - "bootnodes", "borsh", "bufbuild", "bytesize", @@ -60,13 +63,17 @@ "deallocate", "deallocating", "deallocation", + "dedup", "Demultiplexer", "demultiplexing", "Demux", "demuxed", "Deque", + "desynchronized", "DEVNOTE", "Diop", + "disktrie", + "disktries", "distro", "doctest", "doctests", @@ -82,6 +89,7 @@ "forknet", "freelist", "freelists", + "Gbit", "gced", "Ggas", "gprusak", @@ -94,6 +102,7 @@ "Interruptible", "itertools", "itoa", + "jakmeier", "jbajic", "Justfile", "kaiching", @@ -109,8 +118,12 @@ "librocksdb", "lightclient", "lightclients", + "loadtest", + "loadtester", "loadtesting", "localnet", + "locustfile", + "locustfiles", "Logunov", "Mbps", "memtable", @@ -127,11 +140,13 @@ "Millinear", "millis", "mixeddb", + "moar", "mocknet", "multiexp", "multinode", "multishard", "multizip", + "nacl", "nagisa", "nanos", "nanosec", @@ -148,6 +163,7 @@ "nextest", "nikurt", "NOFILE", + "nohup", "Nomicon", "nonexpired", "nonleaf", @@ -155,13 +171,18 @@ "nums", "oneshot", "ords", + "pagodaplatform", + "pathlib", "peekable", "peermanager", "perc", "petagas", "PGAS", + "pids", "pinnable", "pipenv", + "pkill", + "pmap", "postprocesses", "Prefetcher", "prefetchers", @@ -185,6 +206,7 @@ "refcounting", "reindexing", "replayability", + "replaydb", "repr", "reqwest", "reshard", @@ -195,6 +217,9 @@ "respawn", "respawns", "restake", + "restaked", + "restaker", + "Restaking", "retryable", "ripemd", "RISTRETTO", @@ -217,12 +242,15 @@ "SECP", "Seedable", "serde", + "shardnet", "Shreyan", "Shutdownable", + "Skylake", "smallvec", "spammy", "splitdb", "Spurio", + "stdenv", "stdx", "structs", "subaccount", @@ -249,6 +277,9 @@ "TGAS", "thiserror", "threadpool", + "throughputs", + "timespan", + "toplevel", "trisfald", "txns", "typenum", @@ -266,6 +297,7 @@ "unstaking", "unsync", "upto", + "useragent", "usize", "valgrind", "valset", @@ -283,6 +315,7 @@ "xoraddr", "xorshift", "yansi", + "yapf", "yocto", "yoctonear", "zstd", diff --git a/docs/architecture/storage/trie_storage.md b/docs/architecture/storage/trie_storage.md index 76e7429ff89..36cc06185cb 100644 --- a/docs/architecture/storage/trie_storage.md +++ b/docs/architecture/storage/trie_storage.md @@ -107,7 +107,7 @@ collection of old block data which is no longer needed. ### TrieRefcountChange Because we remove unused nodes during garbage collection, we need to track -the reference count (`rc`) for each node. Another reason is that we can de-dup +the reference count (`rc`) for each node. Another reason is that we can dedup values. If the same contract is deployed 1000 times, we only store one contract binary in storage and track its count. diff --git a/docs/practices/workflows/io_trace.md b/docs/practices/workflows/io_trace.md index ef5a4aa82fd..05b54e7fd86 100644 --- a/docs/practices/workflows/io_trace.md +++ b/docs/practices/workflows/io_trace.md @@ -234,11 +234,12 @@ the predecessor account id, presumably to perform some checks. The `sha256` call here is used to shorten implicit account ids. ([Link to code for comparison](https://github.com/sweatco/near-sdk-rs/blob/af6ba3cb75e0bbfc26e346e61aa3a0d1d7f5ac7b/near-contract-standards/src/fungible_token/core_impl.rs#L249-L259)). + Afterwards, a value with 16 bytes (a `u128`) is fetched from the trie state. To serve this, it required reading 30 trie nodes, 19 of them were cached in the accounting cache and were not charged the full gas cost. And the remaining 11 missed the accounting cache but they hit the shard cache. Nothing needed to be -fetched from DB because the SweatCoin specific prefetcher has already loaded +fetched from DB because the Sweatcoin specific prefetcher has already loaded everything into the shard cache. *Note: We see trie node requests despite flat state being used. This is because