fix(consensus)!: optmise foreign pledging for output-only case (#1244)

Description --- fix(consensus): output-only shard groups skip the LocalPrepare and AllPrepare phase and go straight to LocalAccept fix(consensus)!: do not send substate values twice to the same shard groups for prepare and accept phases fix(consensus)!: output pledges are never included in block pledges, taken directly from block evidence fix(consensus): process foreign proposals in on_propose fix(consensus): do not exclude substate pledges that are still pledged to other applicable transactions (Read lock case) fix(consensus): bug causing incorrect failed validation of input pledges fix(consensus): foreign abort reason is carried over to other shards (instead of the unhelpful "ForeignShardDecidedToAbort") fix(consensus): feat(vn/webui): add atom evidence JSON to block view fix(swarm/webui): bug causing UI crash in abort case Motivation and Context --- General stability fixes and optimisations for consensus How Has This Been Tested? --- New consensus unit test Manually tariswap bench, no observed problems Breaking Changes --- - [ ] None - [ ] Requires data directory to be deleted - [x] Other - Please specify BREAKING CHANGE: changes to wire protocol
tari-project · Jan 20, 2025 · b730be3 · b730be3
1 parent 9500de9
commit b730be3
Show file tree

Hide file tree

Showing 55 changed files with 1,423 additions and 781 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,7 +21,7 @@ concurrency:
   cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') || github.ref != 'refs/heads/development' }}
 
 env:
-  nightly_toolchain: nightly-2023-06-12
+  nightly_toolchain: nightly-2025-01-17
   stable_toolchain: 1.84
   CARGO_HTTP_MULTIPLEXING: false
   CARGO_TERM_COLOR: always

diff --git a/applications/tari_dan_app_utilities/src/base_layer_scanner.rs b/applications/tari_dan_app_utilities/src/base_layer_scanner.rs
@@ -48,13 +48,7 @@ use tari_crypto::{
     ristretto::RistrettoPublicKey,
     tari_utilities::{ByteArray, ByteArrayError},
 };
-use tari_dan_common_types::{
-    option::DisplayContainer,
-    optional::Optional,
-    Epoch,
-    NodeAddressable,
-    VersionedSubstateId,
-};
+use tari_dan_common_types::{option::Displayable, optional::Optional, Epoch, NodeAddressable, VersionedSubstateId};
 use tari_dan_storage::{
     consensus_models::{BurntUtxo, SubstateRecord},
     global::{GlobalDb, MetadataKey},

diff --git a/applications/tari_indexer/src/event_scanner.rs b/applications/tari_indexer/src/event_scanner.rs
@@ -280,15 +280,16 @@ impl EventScanner {
             let event_already_exists = tx.event_exists(event_row.clone())?;
             if event_already_exists {
                 // the event was already stored previously
-                warn!(
+                // TODO: Making this debug because it happens a lot and tends to spam the swarm output
+                debug!(
                     target: LOG_TARGET,
-                    "Duplicated event {:}",
+                    "Duplicate {}",
                     data.event
                 );
                 continue;
             }
 
-            info!(
+            debug!(
                 target: LOG_TARGET,
                 "Saving event: {:?}",
                 event_row

diff --git a/applications/tari_swarm_daemon/src/webserver/templates.rs b/applications/tari_swarm_daemon/src/webserver/templates.rs
@@ -62,11 +62,9 @@ pub async fn upload(
 ) -> Result<Json<UploadResponse>, UploadError> {
     let Some(field) = value.next_field().await? else {
         error!("🌐 Upload template: no field found");
-        return Ok(
-            Json(
-                UploadResponse::failure("No multipart file field found".to_string())
-            )
-        );
+        return Ok(Json(UploadResponse::failure(
+            "No multipart file field found".to_string(),
+        )));
     };
 
     let name = field.file_name().unwrap_or("unnamed-template").to_string();

diff --git a/applications/tari_swarm_daemon/webui/src/routes/Main.tsx b/applications/tari_swarm_daemon/webui/src/routes/Main.tsx
@@ -167,7 +167,7 @@ function ExtraInfoVN({ name, url, addTxToPool, autoRefresh, state, horizontal }:
                 <div onClick={() => copyToClipboard(tx)}>{copied == tx ? "Copied" : shorten(tx)}</div>
                 <div style={{ color: known ? "green" : "red" }}><b>{known && "Yes" || "No"}</b></div>
                 <div>{abort_details || <i>unknown</i>}</div>
-                <div>{final_decision || <i>unknown</i>}</div>
+                <div>{("Abort" in final_decision) ? <>Abort ({final_decision.Abort})</> : <>Commit</>}</div>
               </>
             );
           })}

diff --git a/applications/tari_validator_node/src/p2p/services/mempool/service.rs b/applications/tari_validator_node/src/p2p/services/mempool/service.rs
@@ -273,9 +273,9 @@ where TValidator: Validator<Transaction, Context = (), Error = TransactionValida
                 .await
                 .map_err(|_| MempoolError::ConsensusChannelClosed)?;
 
-            // If we received the message from our local shard group, we don't need to gossip it again on the topic
-            // (prevents Duplicate errors)
-            if sender_shard_group.map_or(true, |sg| sg != local_committee_shard.shard_group()) {
+            // If we received the message from gossip (sender_shard_group is Some), we don't need to gossip it again on
+            // the topic (prevents Duplicate errors)
+            if sender_shard_group.is_none() {
                 // This validator is involved, we to send the transaction to local replicas
                 if let Err(e) = self
                     .gossip

diff --git a/applications/tari_validator_node_cli/src/command/transaction.rs b/applications/tari_validator_node_cli/src/command/transaction.rs
@@ -30,7 +30,7 @@ use std::{
 use anyhow::anyhow;
 use clap::{Args, Subcommand};
 use tari_dan_common_types::{
-    option::{DisplayCont, DisplayContainer},
+    option::{DisplayContainer, Displayable},
     optional::Optional,
     SubstateAddress,
     SubstateRequirement,
@@ -498,7 +498,7 @@ fn summarize_finalize_result(finalize: &FinalizeResult) {
 }
 
 fn display_vec<W: fmt::Write>(writer: &mut W, ty: &Type, result: &InstructionResult) -> fmt::Result {
-    fn display_slice<T: fmt::Display>(slice: &[T]) -> DisplayCont<&[T]> {
+    fn display_slice<T: fmt::Display>(slice: &[T]) -> DisplayContainer<&[T]> {
         slice.display()
     }
 

diff --git a/applications/tari_validator_node_web_ui/src/Components/CodeBlock.tsx b/applications/tari_validator_node_web_ui/src/Components/CodeBlock.tsx
@@ -21,7 +21,7 @@
 //  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import { IoExpandOutline } from "react-icons/io5";
-import { useState } from "react";
+import { ReactNode, useState } from "react";
 import Dialog from "@mui/material/Dialog";
 import IconButton from "@mui/material/IconButton";
 import Typography from "@mui/material/Typography";
@@ -31,7 +31,13 @@ import { CodeBlock } from "./StyledComponents";
 import useMediaQuery from "@mui/material/useMediaQuery";
 import { useTheme } from "@mui/material/styles";
 
-export default function CodeBlockExpand({ title, children }: any) {
+interface Props {
+  title: string;
+  children?: ReactNode;
+  contentsWhenUnexpanded?: ReactNode;
+}
+
+export default function CodeBlockDialog({ title, children, contentsWhenUnexpanded = false }: Props) {
   const [open, setOpen] = useState(false);
   const theme = useTheme();
   const matches = useMediaQuery(theme.breakpoints.down("md"));
@@ -46,12 +52,7 @@ export default function CodeBlockExpand({ title, children }: any) {
 
   return (
     <div>
-      <CodeBlock
-        style={{
-          position: "relative",
-        }}
-      >
-        {children}
+      {contentsWhenUnexpanded ? (
         <IconButton
           onClick={handleClickOpen}
           style={{
@@ -60,9 +61,27 @@ export default function CodeBlockExpand({ title, children }: any) {
             float: "right",
           }}
         >
+          {contentsWhenUnexpanded}
           <IoExpandOutline style={{ height: 16, width: 16 }} />
         </IconButton>
-      </CodeBlock>
+      ) : (
+        <CodeBlock
+          style={{
+            position: "relative",
+          }}
+        >
+          {children}
+          <IconButton
+            onClick={handleClickOpen}
+            style={{
+              position: "sticky",
+              bottom: "0",
+              float: "right",
+            }}
+          >
+            <IoExpandOutline style={{ height: 16, width: 16 }} />
+          </IconButton>
+        </CodeBlock>)}
       <Dialog fullScreen={matches} open={open} onClose={handleClose} maxWidth="xl" fullWidth>
         <Box
           style={{

diff --git a/applications/tari_validator_node_web_ui/src/routes/Blocks/BlockDetails.tsx b/applications/tari_validator_node_web_ui/src/routes/Blocks/BlockDetails.tsx
@@ -185,7 +185,7 @@ export default function BlockDetails() {
                           </TableRow>
                           <TableRow>
                             <TableCell>Epoch</TableCell>
-                            <DataTableCell>{block!.header.epoch}</DataTableCell>
+                            <DataTableCell>{block!.header.epoch} (ShardGroup {block!.header.shard_group.start}-{block!.header.shard_group.end_inclusive})</DataTableCell>
                           </TableRow>
                           <TableRow>
                             <TableCell>Height</TableCell>

diff --git a/applications/tari_validator_node_web_ui/src/routes/Blocks/Transactions.tsx b/applications/tari_validator_node_web_ui/src/routes/Blocks/Transactions.tsx
@@ -25,6 +25,8 @@ import { Table, TableBody, TableCell, TableContainer, TableHead, TableRow } from
 import StatusChip from "../../Components/StatusChip";
 import type { TransactionAtom } from "@tari-project/typescript-bindings";
 import { Link } from "react-router-dom";
+import { renderJson } from "../../utils/helpers";
+import CodeBlockDialog from "../../Components/CodeBlock";
 
 function Transaction({ transaction }: { transaction: TransactionAtom }) {
   const decision = typeof transaction.decision === "object" ? "Abort" : "Commit";
@@ -36,6 +38,11 @@ function Transaction({ transaction }: { transaction: TransactionAtom }) {
       <TableCell>
         <StatusChip status={decision} />
       </TableCell>
+      <TableCell>
+        <CodeBlockDialog title="Evidence" contentsWhenUnexpanded={<></>}>
+          {renderJson(transaction.evidence)}
+        </CodeBlockDialog>
+      </TableCell>
       <TableCell>{transaction.leader_fee?.fee}</TableCell>
       <TableCell>{transaction.transaction_fee}</TableCell>
     </TableRow>
@@ -50,6 +57,7 @@ export default function Transactions({ transactions }: { transactions: Transacti
           <TableRow>
             <TableCell>Transaction ID</TableCell>
             <TableCell>Decision</TableCell>
+            <TableCell>Evidence</TableCell>
             <TableCell>Leader fee</TableCell>
             <TableCell>Transaction fee</TableCell>
           </TableRow>

diff --git a/applications/tari_validator_node_web_ui/src/routes/Transactions/Events.tsx b/applications/tari_validator_node_web_ui/src/routes/Transactions/Events.tsx
@@ -28,7 +28,7 @@ import CopyToClipboard from "../../Components/CopyToClipboard";
 import { renderJson } from "../../utils/helpers";
 import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown";
 import KeyboardArrowUpIcon from "@mui/icons-material/KeyboardArrowUp";
-import CodeBlockExpand from "../../Components/CodeBlock";
+import CodeBlockDialog from "../../Components/CodeBlock";
 import type { Event } from "@tari-project/typescript-bindings";
 
 function RowData({ substate_id, template_address, topic, tx_hash, payload }: Event, index: number) {
@@ -67,7 +67,7 @@ function RowData({ substate_id, template_address, topic, tx_hash, payload }: Eve
       <TableRow>
         <DataTableCell style={{ paddingBottom: 0, paddingTop: 0 }} colSpan={5}>
           <Collapse in={open} timeout="auto" unmountOnExit>
-            <CodeBlockExpand title="Payload">{renderJson(payload)}</CodeBlockExpand>
+            <CodeBlockDialog title="Payload">{renderJson(payload)}</CodeBlockDialog>
           </Collapse>
         </DataTableCell>
       </TableRow>

diff --git a/applications/tari_validator_node_web_ui/src/routes/Transactions/FeeInstructions.tsx b/applications/tari_validator_node_web_ui/src/routes/Transactions/FeeInstructions.tsx
@@ -26,7 +26,7 @@ import { DataTableCell, AccordionIconButton } from "../../Components/StyledCompo
 import { renderJson } from "../../utils/helpers";
 import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown";
 import KeyboardArrowUpIcon from "@mui/icons-material/KeyboardArrowUp";
-import CodeBlockExpand from "../../Components/CodeBlock";
+import CodeBlockDialog from "../../Components/CodeBlock";
 import type { Instruction } from "@tari-project/typescript-bindings";
 
 function RowData({ title, data, index }: { title: string; data: Instruction; index: number }) {
@@ -51,7 +51,7 @@ function RowData({ title, data, index }: { title: string; data: Instruction; ind
       <TableRow key={`${index}-2`}>
         <DataTableCell style={{ paddingBottom: 0, paddingTop: 0 }} colSpan={2}>
           <Collapse in={open} timeout="auto" unmountOnExit>
-            <CodeBlockExpand title={title}>{renderJson(data)}</CodeBlockExpand>
+            <CodeBlockDialog title={title}>{renderJson(data)}</CodeBlockDialog>
           </Collapse>
         </DataTableCell>
       </TableRow>

diff --git a/applications/tari_validator_node_web_ui/src/routes/Transactions/Instructions.tsx b/applications/tari_validator_node_web_ui/src/routes/Transactions/Instructions.tsx
@@ -26,7 +26,7 @@ import { DataTableCell, AccordionIconButton } from "../../Components/StyledCompo
 import { renderJson } from "../../utils/helpers";
 import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown";
 import KeyboardArrowUpIcon from "@mui/icons-material/KeyboardArrowUp";
-import CodeBlockExpand from "../../Components/CodeBlock";
+import CodeBlockDialog from "../../Components/CodeBlock";
 import type { Instruction } from "@tari-project/typescript-bindings";
 
 function RowData({ title, data }: { title: string; data: Instruction }, index: number) {
@@ -51,7 +51,7 @@ function RowData({ title, data }: { title: string; data: Instruction }, index: n
       <TableRow key={`${index}-2`}>
         <DataTableCell style={{ paddingBottom: 0, paddingTop: 0 }} colSpan={2}>
           <Collapse in={open} timeout="auto" unmountOnExit>
-            <CodeBlockExpand title={title}>{renderJson(data)}</CodeBlockExpand>
+            <CodeBlockDialog title={title}>{renderJson(data)}</CodeBlockDialog>
           </Collapse>
         </DataTableCell>
       </TableRow>

diff --git a/applications/tari_validator_node_web_ui/src/utils/helpers.tsx b/applications/tari_validator_node_web_ui/src/utils/helpers.tsx
@@ -24,10 +24,8 @@ import { toHexString } from "../routes/VN/Components/helpers";
 import type { ShardGroup, SubstateId } from "@tari-project/typescript-bindings";
 
 export const renderJson = (json: any) => {
-  if (!json) {
-    return <span>Null</span>;
-  }
-  if (Array.isArray(json)) {
+
+  if (json && Array.isArray(json)) {
     //eslint-disable-next-line eqeqeq
     if (json.length == 32) {
       return <span className="string">"{toHexString(json)}"</span>;
@@ -43,7 +41,12 @@ export const renderJson = (json: any) => {
         ],
       </>
     );
-  } else if (typeof json === "object") {
+  }
+
+  if (typeof json === "object") {
+    if (!json) {
+      return <span>null</span>;
+    }
     return (
       <>
         {"{"}
@@ -57,10 +60,12 @@ export const renderJson = (json: any) => {
         {"}"}
       </>
     );
-  } else {
-    if (typeof json === "string") return <span className="string">"{json}"</span>;
-    return <span className="other">{json}</span>;
   }
+
+  if (typeof json === "string") return <span className="string">"{json}"</span>;
+  if (typeof json === "number") return <span className="number">"{json}"</span>;
+  if (typeof json === "boolean") return <span className="boolean">{json ? "true" : "false"}</span>;
+  return <span className="other">{json}</span>;
 };
 
 export function fromHexString(hexString: string) {

diff --git a/dan_layer/common_types/src/committee.rs b/dan_layer/common_types/src/committee.rs
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use tari_common_types::types::PublicKey;
 use tari_engine_types::substate::SubstateId;
 
-use crate::{NumPreshards, ShardGroup, SubstateAddress};
+use crate::{Epoch, NumPreshards, ShardGroup, SubstateAddress};
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Default, Hash)]
 #[cfg_attr(
@@ -180,6 +180,7 @@ pub struct CommitteeInfo {
     num_shard_group_members: u32,
     num_committees: u32,
     shard_group: ShardGroup,
+    epoch: Epoch,
 }
 
 impl CommitteeInfo {
@@ -188,15 +189,22 @@ impl CommitteeInfo {
         num_shard_group_members: u32,
         num_committees: u32,
         shard_group: ShardGroup,
+        epoch: Epoch,
     ) -> Self {
         Self {
             num_shards,
             num_shard_group_members,
             num_committees,
             shard_group,
+            epoch,
         }
     }
 
+    /// Returns the epoch of this CommitteeInfo.
+    pub fn epoch(&self) -> Epoch {
+        self.epoch
+    }
+
     /// Returns $n - f$ (i.e $2f + 1$) where n is the number of committee members and f is the tolerated failure nodes.
     pub fn quorum_threshold(&self) -> u32 {
         self.num_shard_group_members - self.max_failures()