Feature/plt scheduler port

.github/workflows/build-test.yaml

@@ -34,6 +34,7 @@ on:
     - 'concordium-consensus/smart-contracts'
     - 'concordium-consensus/haskell-lmdb'
     - 'concordium-node/rustfmt.toml'
+    - 'plt'
 
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
@@ -48,17 +49,19 @@ on:
     - 'concordium-consensus/smart-contracts'
     - 'concordium-consensus/haskell-lmdb'
     - 'concordium-node/rustfmt.toml'
+    - 'plt'
   workflow_dispatch: # allow manual trigger
 
 env:
   dummy: 17                # change to force cache invalidation
   CARGO_TERM_COLOR: always # implicitly adds '--color=always' to all cargo commands
+  RUST_VERSION: 1.86
+  GHC_VERSION: 9.10.2
 
 jobs:
 
   fourmolu:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
     steps:
 
     - name: Download fourmolu
@@ -93,25 +96,17 @@ jobs:
 
   rustfmt:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
-
-    strategy:
-      matrix:
-        plan:
-        - rust: "1.82"
-
     steps:
     - name: Checkout
       uses: actions/checkout@v2
       with:
-        # token: ${{ secrets.CONCORDIUM_CI }}
         submodules: recursive
 
     - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:
         profile: minimal
-        toolchain: ${{ matrix.plan.rust }}
+        toolchain: ${{ env.RUST_VERSION }}
         override: true
         components: rustfmt
 
@@ -127,13 +122,6 @@ jobs:
     needs: [fourmolu, rustfmt]
     # Use fixed OS version because we install packages on the system.
     runs-on: ubuntu-22.04
-    if: ${{ !github.event.pull_request.draft }}
-
-    strategy:
-      matrix:
-        plan:
-        - rust: 1.82
-          ghc: 9.10.2
 
     steps:
     - name: Remove unnecessary files
@@ -164,7 +152,7 @@ jobs:
       uses: actions-rs/toolchain@v1
       with:
         profile: minimal
-        toolchain: ${{ matrix.plan.rust }}
+        toolchain: ${{ env.RUST_VERSION }}
         override: true
         components: clippy, llvm-tools
         target: x86_64-pc-windows-gnu
@@ -184,10 +172,10 @@ jobs:
           concordium-base/smart-contracts/lib
           concordium-node/target
           collector/target
-        key: ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}-${{ hashFiles('concordium-base/rust-src/**/*.rs','concordium-base/smart-contracts/wasm-chain-integration/**/*.rs')}}
+        key: ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ env.RUST_VERSION }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}-${{ hashFiles('concordium-base/rust-src/**/*.rs','concordium-base/smart-contracts/wasm-chain-integration/**/*.rs')}}
         restore-keys: |
-          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
-          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}
+          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ env.RUST_VERSION }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          ${{ runner.os }}-${{ env.dummy }}-rust-deps-${{ env.RUST_VERSION }}
 
     # HASKELL #
 
@@ -208,9 +196,9 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/.stack
-        key: ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}
         restore-keys: |
-          ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ matrix.plan.ghc }}
+          ${{ runner.os }}-${{ env.dummy }}-stack-global-${{ env.GHC_VERSION }}
     - name: Cache '.stack-work'
       uses: actions/cache@v4
       with:
@@ -220,11 +208,11 @@ jobs:
           concordium-consensus/.stack-work
           concordium-consensus/haskell-lmdb/.stack-work
 
-        key: ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}-${{ steps.cache-keys.outputs.proto_hash }}-${{ steps.cache-keys.outputs.base_hash }}
+        key: ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}-${{ steps.cache-keys.outputs.proto_hash }}-${{ steps.cache-keys.outputs.base_hash }}
         restore-keys: |
-          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}-${{ steps.cache-keys.outputs.proto_hash }}-
-          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}-
-          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}
+          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}-${{ steps.cache-keys.outputs.proto_hash }}-
+          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}-${{ hashFiles('**.yaml') }}-
+          ${{ runner.os }}-${{ env.dummy }}-stack-work-${{ env.GHC_VERSION }}
 
     # Compile Haskell sources. This must be done before running checks or tests on the Rust sources.
     - name: Build consensus and run tests

.github/workflows/plt-build-test.yaml

@@ -0,0 +1,51 @@
+name: PLT scheduler build and run tests
+
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - '.github/workflows/plt-build-test.yaml'
+    - 'concordium-base'
+    - 'plt'
+
+  pull_request:
+    paths:
+    - '.github/workflows/plt-build-test.yaml'
+    - 'concordium-base'
+    - 'plt'
+env:
+  CARGO_TERM_COLOR: always # implicitly adds '--color=always' to all cargo commands
+
+jobs:
+  rustfmt:
+    name: Check formatting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Run rustfmt
+        working-directory: plt
+        run: |
+          rustup component add rustfmt
+          cargo fmt --check
+
+  clippy_test:
+    name: Run Clippy and tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Clippy
+        working-directory: plt
+        run: |
+          rustup component add clippy
+          cargo clippy --locked --all-targets --all-features -- -D warnings
+      - name: Test
+        working-directory: plt
+        run: |
+          cargo test --locked --all-targets --all-features

.github/workflows/release.yaml

@@ -22,7 +22,7 @@ on:
 env:
   UBUNTU_VERSION: '22.04'
   STATIC_LIBRARIES_IMAGE_TAG: 'rust-1.82_ghc-9.10.2'
-  RUST_VERSION: '1.82'
+  RUST_VERSION: '1.86'
   STACK_VERSION: '3.7.1'
   FLATBUFFERS_VERSION: '23.5.26'
   GHC_VERSION: '9.10.2'

.gitignore

@@ -27,7 +27,6 @@ deps/internal/crypto/build
 *.ps
 scripts/static-libs/static-libs-docker/output_dir/
 result
-**/**/Cargo.lock
 0
 **/**/0
 scripts/genesis-data/genesis_data/**
@@ -49,6 +48,7 @@ genesis_data
 /concordium-consensus/a.out
 /concordium-consensus/HSdll.dll
 /concordium-consensus/HSdll.dll.a
+/concordium-consensus/lib
 concordium-node/deps/static-libs
 .dir-locals.el
 hie.yaml

CHANGELOG.md

@@ -8,6 +8,7 @@
   the token module state. They are still required by the current token module implementation, and initialization
   without the parameters set will be rejected, so there are no observable changes to PLT behaviour.
 - Fixed the `build_catchup_url` in the Ubuntu build release pipeline.
+- Added boilerplate code for the upcoming P10.
 
 ## 9.0.7
 

collector/src/bin/collector.rs

@@ -11,7 +11,11 @@ use tonic::transport::{channel::Channel, ClientTlsConfig};
 #[macro_use]
 extern crate log;
 
-#[allow(clippy::large_enum_variant, clippy::enum_variant_names)]
+#[allow(
+    clippy::large_enum_variant,
+    clippy::enum_variant_names,
+    clippy::doc_overindented_list_items
+)]
 mod grpc {
     mod plt {
         tonic::include_proto!("concordium.v2.plt");
@@ -208,7 +212,7 @@ async fn main() {
 }
 
 #[allow(clippy::cognitive_complexity)]
-async fn collect_data<'a>(
+async fn collect_data(
     node_name: NodeName,
     grpc_host: String,
     conf: &ConfigCli,

concordium-consensus/Setup.hs

@@ -4,52 +4,74 @@ import Distribution.Simple.LocalBuildInfo
 import Distribution.Simple.Setup
 import Distribution.Simple.Utils
 import Distribution.System
+import Distribution.Verbosity
 
 import System.Directory
 import System.Environment
 
 import Data.Maybe
 
-smartContractRoot = "../concordium-base/smart-contracts"
+-- | Notify and execute a command, if fails exit with the same exit code.
+runCmd :: Verbosity -> String -> IO ()
+runCmd verbosity cmd = do
+    notice verbosity $ "Running '" ++ cmd ++ "'"
+    let command : args = words cmd
+    rawSystemExit verbosity command $ args
 
-makeRust :: Args -> ConfigFlags -> IO HookedBuildInfo
-makeRust args flags = do
+-- | Path to Concordium Smart Contract Engine rust crate relative to this file.
+smartContractEngineCrateRelative = "../concordium-base/smart-contracts/wasm-chain-integration"
+
+-- | Path to plt rust workspace relative to this file.
+pltWorkspaceRelative = "../plt"
+
+preConfHook :: Args -> ConfigFlags -> IO HookedBuildInfo
+preConfHook args flags = do
     let verbosity = fromFlag $ configVerbosity flags
-    rawSystemExit verbosity "mkdir" ["-p", smartContractRoot ++ "/lib"]
 
-    -- This way of determining the platform is not ideal.
-    notice verbosity "Calling 'cargo build'"
-    rawSystemExit
-        verbosity
-        "cargo"
-        ["build", "--release", "--manifest-path", smartContractRoot ++ "/wasm-chain-integration/Cargo.toml", "--features=enable-ffi"]
+    -- Convert relative paths into absolute paths.
+    libraryDestination <- canonicalizePath "./lib"
+    -- Ensure destination directory exists.
+    runCmd verbosity $ "mkdir -p " ++ libraryDestination
+
+    -- Build and copy/symlink Concordium Smart contract Engine library.
+    smartContractEngineCrate <- canonicalizePath smartContractEngineCrateRelative
+    runCmd verbosity $ "cargo build --release --locked --features=enable-ffi --manifest-path=" ++ smartContractEngineCrate ++ "/Cargo.toml"
+    case buildOS of
+        Windows -> do
+            runCmd verbosity $ "cp -u " ++ smartContractEngineCrate ++ "/target/release/concordium_smart_contract_engine.dll " ++ libraryDestination
+        OSX -> do
+            runCmd verbosity $ "ln -s -f " ++ smartContractEngineCrate ++ "/target/release/libconcordium_smart_contract_engine.a " ++ libraryDestination
+            runCmd verbosity $ "ln -s -f " ++ smartContractEngineCrate ++ "/target/release/libconcordium_smart_contract_engine.dylib " ++ libraryDestination
+        _ -> do
+            runCmd verbosity $ "ln -s -f " ++ smartContractEngineCrate ++ "/target/release/libconcordium_smart_contract_engine.a " ++ libraryDestination
+            runCmd verbosity $ "ln -s -f " ++ smartContractEngineCrate ++ "/target/release/libconcordium_smart_contract_engine.so " ++ libraryDestination
+
+    -- Build and copy/symlink PLT scheduler project
+    pltWorkspace <- canonicalizePath pltWorkspaceRelative
+    runCmd verbosity $ "cargo build --release --locked --features ffi -p plt-scheduler --manifest-path=" ++ pltWorkspace ++ "/Cargo.toml"
     case buildOS of
         Windows -> do
-            notice verbosity "Copying concordium_smart_contract_engine library"
-            rawSystemExit verbosity "cp" ["-u", smartContractRoot ++ "/wasm-chain-integration/target/release/concordium_smart_contract_engine.dll", smartContractRoot ++ "/lib/"]
-            -- We remove the static library if it exists. Previously, it would have been copied
-            -- over, but now we want to just link with the dynamic library, so we ensure it is
-            -- removed.
-            rawSystemExit verbosity "rm" ["-f", smartContractRoot ++ "/lib/libconcordium_smart_contract_engine.a"]
+            runCmd verbosity $ "cp -u " ++ pltWorkspace ++ "/target/release/plt_scheduler.dll " ++ libraryDestination
+        OSX -> do
+            runCmd verbosity $ "ln -s -f " ++ pltWorkspace ++ "/target/release/libplt_scheduler.a " ++ libraryDestination
+            runCmd verbosity $ "ln -s -f " ++ pltWorkspace ++ "/target/release/libplt_scheduler.dylib " ++ libraryDestination
         _ -> do
-            rawSystemExit verbosity "ln" ["-s", "-f", "../wasm-chain-integration/target/release/libconcordium_smart_contract_engine.a", smartContractRoot ++ "/lib/"]
-            case buildOS of
-                OSX ->
-                    rawSystemExit verbosity "ln" ["-s", "-f", "../wasm-chain-integration/target/release/libconcordium_smart_contract_engine.dylib", smartContractRoot ++ "/lib/libconcordium_smart_contract_engine.dylib"]
-                _ ->
-                    rawSystemExit verbosity "ln" ["-s", "-f", "../wasm-chain-integration/target/release/libconcordium_smart_contract_engine.so", smartContractRoot ++ "/lib/libconcordium_smart_contract_engine.so"]
+            runCmd verbosity $ "ln -s -f " ++ pltWorkspace ++ "/target/release/libplt_scheduler.a " ++ libraryDestination
+            runCmd verbosity $ "ln -s -f " ++ pltWorkspace ++ "/target/release/libplt_scheduler.so " ++ libraryDestination
     return emptyHookedBuildInfo
 
 -- | On Windows, copy the DLL files to the binary install directory. This is to ensure that they
 -- are accessible when running the binaries, tests and benchmarks.
-copyDlls :: Args -> CopyFlags -> PackageDescription -> LocalBuildInfo -> IO ()
-copyDlls _ flags pkgDescr lbi = case buildOS of
+postCopyHook :: Args -> CopyFlags -> PackageDescription -> LocalBuildInfo -> IO ()
+postCopyHook _ flags pkgDescr lbi = case buildOS of
     Windows -> do
         let installDirs = absoluteComponentInstallDirs pkgDescr lbi (localUnitId lbi) copydest
-        let copyLib lib = do
-                rawSystemExit verbosity "cp" ["-u", smartContractRoot ++ "/lib/" ++ lib ++ ".dll", bindir installDirs]
-                notice verbosity $ "Copy " ++ lib ++ " to " ++ bindir installDirs
-        copyLib "concordium_smart_contract_engine"
+        -- Copy DLL for Concordium Smart Contract Engine
+        smartContractEngineCrate <- canonicalizePath smartContractEngineCrateRelative
+        runCmd verbosity $ "cp -u " ++ smartContractEngineCrate ++ "/target/release/concordium_smart_contract_engine.dll " ++ bindir installDirs
+        -- Copy DLL for PLT scheduler
+        pltWorkspace <- canonicalizePath pltWorkspaceRelative
+        runCmd verbosity $ "cp -u " ++ pltWorkspace ++ "/target/release/plt_scheduler.dll " ++ bindir installDirs
     _ -> return ()
   where
     distPref = fromFlag (copyDistPref flags)
@@ -59,6 +81,6 @@ copyDlls _ flags pkgDescr lbi = case buildOS of
 main =
     defaultMainWithHooks $
         simpleUserHooks
-            { preConf = makeRust,
-              postCopy = copyDlls
+            { preConf = preConfHook,
+              postCopy = postCopyHook
             }

concordium-consensus/package.yaml

@@ -88,7 +88,9 @@ library:
       - -O2
       - -fno-ignore-asserts
 
-  extra-libraries: concordium_smart_contract_engine
+  extra-libraries:
+      - concordium_smart_contract_engine
+      - plt_scheduler
 
   when:
     - condition: "!(os(windows)) && !(flag(dynamic))"

concordium-consensus/src/Concordium/GlobalState/BakerInfo.hs

@@ -377,6 +377,7 @@ genesisBakerInfoEx spv cp GenesisBaker{..} = case spv of
     SP7 -> binfoV1
     SP8 -> binfoV1
     SP9 -> binfoV1
+    SP10 -> binfoV1
   where
     bkrInfo =
         BakerInfo

concordium-consensus/src/Concordium/GlobalState/Block.hs

@@ -144,6 +144,7 @@ blockVersion SP6 = 3
 blockVersion SP7 = 3
 blockVersion SP8 = 3
 blockVersion SP9 = 3
+blockVersion SP10 = 3
 {-# INLINE blockVersion #-}
 
 -- | Type class that supports serialization of a block.

concordium-consensus/src/Concordium/GlobalState/Persistent/Account.hs

@@ -865,6 +865,7 @@ migratePersistentAccount m@StateMigrationParametersP5ToP6{} (PAV2 acc) = PAV2 <$
 migratePersistentAccount m@StateMigrationParametersP6ToP7{} (PAV2 acc) = PAV3 <$> V1.migratePersistentAccount m acc
 migratePersistentAccount m@StateMigrationParametersP7ToP8{} (PAV3 acc) = PAV4 <$> V1.migratePersistentAccount m acc
 migratePersistentAccount m@StateMigrationParametersP8ToP9{} (PAV4 acc) = PAV5 <$> V1.migratePersistentAccount m acc
+migratePersistentAccount m@StateMigrationParametersP9ToP10{} (PAV5 acc) = PAV5 <$> V1.migratePersistentAccount m acc
 
 -- | Migrate a 'PersistentBakerInfoRef' between protocol versions according to a state migration.
 migratePersistentBakerInfoRef ::
@@ -887,6 +888,7 @@ migratePersistentBakerInfoRef m@StateMigrationParametersP5ToP6{} (PBIRV2 bir) =
 migratePersistentBakerInfoRef m@StateMigrationParametersP6ToP7{} (PBIRV2 bir) = PBIRV3 <$> V1.migratePersistentBakerInfoEx m bir
 migratePersistentBakerInfoRef m@StateMigrationParametersP7ToP8{} (PBIRV3 bir) = PBIRV4 <$> V1.migratePersistentBakerInfoEx m bir
 migratePersistentBakerInfoRef m@StateMigrationParametersP8ToP9{} (PBIRV4 bir) = PBIRV5 <$> V1.migratePersistentBakerInfoEx m bir
+migratePersistentBakerInfoRef m@StateMigrationParametersP9ToP10{} (PBIRV5 bir) = PBIRV5 <$> V1.migratePersistentBakerInfoEx m bir
 
 -- * Conversion