feat(healthcheck): add healthcheck module with liveness and readiness probes

checks/default.nix

@@ -3,5 +3,6 @@
   imports = [
     ./packages-ci-matrix.nix
     ./pre-commit.nix
+    ./vm-tests.nix
   ];
 }

checks/vm-tests.nix

@@ -0,0 +1,39 @@
+{
+  lib,
+  inputs,
+  self,
+  ...
+}:
+{
+  perSystem =
+    {
+      inputs',
+      self',
+      pkgs,
+      system,
+      ...
+    }:
+    {
+      checks = {
+        "healthcheck-test-01" = pkgs.testers.runNixOSTest {
+          name = "healthcheck-test-01";
+          node.specialArgs = {
+            inherit self system lib;
+          };
+          testScript = ''
+            machine.start()
+            machine.wait_for_unit("default.target")
+          '';
+          nodes.machine =
+            { pkgs, ... }:
+            {
+              imports = [
+                ../machines/modules/base.nix
+                ../machines/modules/healthcheck.nix
+              ];
+            };
+
+        };
+      };
+    };
+}

flake.nix

@@ -221,6 +221,7 @@
         ./checks
         ./modules
         ./packages
+        ./machines
         ./shells
       ];
       systems = [

machines/default.nix

@@ -0,0 +1,43 @@
+{
+  lib,
+  self,
+  ...
+}:
+let
+  system = "x86_64-linux";
+  pkgs = import self.inputs.nixpkgs {
+    inherit system;
+    config.allowUnfree = true;
+  };
+  mkMachine = x: {
+    "${lib.removeSuffix ".nix" x}" = lib.nixosSystem {
+      specialArgs = { inherit self system; };
+      modules = lib.unique [
+        {
+          nixpkgs.system = system;
+        }
+        ./modules/base.nix
+        ./modules/${x}
+      ];
+    };
+  };
+in
+{
+  flake.nixosConfigurations = (
+    lib.mergeAttrsList (lib.map (x: mkMachine x) (builtins.attrNames (builtins.readDir ./modules)))
+  );
+  flake.modules.nixos = (
+    lib.mergeAttrsList (
+      lib.map (x: {
+        "machine_${lib.removeSuffix ".nix" x}" = (
+          import (./modules + "/${x}") {
+            inherit
+              self
+              pkgs
+              ;
+          }
+        );
+      }) (builtins.attrNames (builtins.readDir ./modules))
+    )
+  );
+}

machines/modules/all.nix

@@ -0,0 +1,18 @@
+{
+  self,
+  lib,
+  system,
+  ...
+}:
+{
+  imports = lib.map (x: ./. + "/${x}") (
+    builtins.attrNames (
+      lib.removeAttrs (builtins.readDir ./.) [
+        "base.nix"
+        "all.nix"
+      ]
+    )
+  );
+
+  environment.systemPackages = lib.attrValues self.packages."${system}";
+}

machines/modules/base.nix

@@ -0,0 +1,14 @@
+{ lib, ... }:
+{
+  fileSystems = {
+    "/".device = lib.mkDefault "/dev/sda";
+  };
+  boot.loader.grub.devices = lib.mkDefault [ "/dev/sda" ];
+  virtualisation.vmVariant = {
+    virtualization = {
+      diskSize = 10 * 1024 * 1024 * 1024; # 10GB
+      memorySize = 8192; # 8GB
+      cores = 4;
+    };
+  };
+}

machines/modules/ethereum-validators-monitoring.nix

@@ -0,0 +1,20 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.ethereum-validators-monitoring
+  ];
+
+  services.ethereum-validators-monitoring = {
+    db = {
+      host = "http://localhost:8123/";
+      user = "ethereum";
+      password-file = ../blankpass.txt;
+      name = "ethereum";
+    };
+    instances = {
+      # The Ethereum Validator Monitoring sends out too many requests to servers,
+      # which causes attestations to be missing. Therefore, as we lack a
+      # dedicated server, we are unable to have e-v-m.
+    };
+  };
+}

machines/modules/folder-size-metrics.nix

@@ -0,0 +1,15 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.folder-size-metrics
+  ];
+
+  services.folder-size-metrics = {
+    enable = true;
+    # args = { #Unchanged
+    #   port = 8888;
+    #   base-path = "/var/lib";
+    #   interval-sec = 60;
+    # };
+  };
+}

machines/modules/healthcheck.nix

@@ -0,0 +1,53 @@
+{ self, pkgs, ... }:
+{
+  imports = [
+    self.modules.nixos.healthcheck
+  ];
+
+  systemd.services.test = {
+    description = "";
+    enable = true;
+    path = [
+    ];
+    serviceConfig = {
+      Type = "simple";
+      Restart = "on-failure";
+      RestartSec = "10s";
+
+      ExecStart = ''
+        sleep 10; # Simulate a startup delay.
+        echo "Starting test service...";
+        echo "Test" > /tmp/test.log
+        while true ; do
+          echo "Test" | nc -l 8080
+        done
+      '';
+    };
+  };
+
+  # --- Health Check Configuration ---
+  mcl.services.test.healthcheck = {
+    runtimePackages = with pkgs; [
+      netcat
+      curl
+    ];
+
+    # READINESS: Use the notify pattern to signal when the service is truly ready.
+    readiness-probe = {
+      enable = true;
+      command = "ls /tmp/test.log";
+      interval = 2;
+      statusWaitingMessage = "Test starting, waiting...";
+      statusReadyMessage = "Test is ready.";
+    };
+
+    # LIVENESS: After startup, use a timer to periodically check health.
+    liveness-probe = {
+      enable = true;
+      command = "[ \"$(nc -w 2 localhost 8080)\" = \"Test2\" ]";
+      initialDelay = 15;
+      interval = 30; # Check every 30 seconds.
+      timeout = 5;
+    };
+  };
+}

machines/modules/lido-keys-api.nix

@@ -0,0 +1,30 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.lido-keys-api
+  ];
+
+  services.lido-keys-api = {
+    enable = true;
+    args = {
+      port = 3000;
+      cors-whitelist-regexp = "^https?://(?:.+?\.)?(?:lido|testnet|mainnet|holesky)\.fi$";
+      global-throttle-ttl = 5;
+      global-throttle-limit = 100;
+      global-cache-ttl = 1;
+      sentry-dsn = "";
+      log-level = "debug";
+      log-format = "json";
+      db-name = "node_operator_keys_service_db";
+      db-port = 5432;
+      db-host = "127.0.0.1";
+      db-user = "postgres";
+      db-password = "";
+      provider-json-rpc-max-batch-size = 100;
+      provider-concurrent-requests = 5;
+      provider-batch-aggregation-wait-ms = 10;
+      validator-registry-enable = true;
+    };
+  };
+
+}

machines/modules/lido-validator-ejector.nix

@@ -0,0 +1,28 @@
+{ self, config, ... }:
+{
+  imports = [
+    self.modules.nixos.lido-validator-ejector
+  ];
+
+  services.lido-validator-ejector = {
+    enable = true;
+    args = {
+      messages-location = config.services.lido-withdrawals-automation.args.output-folder;
+      consensus-node = "";
+      execution-node = "";
+      locator-address = "";
+      blocks-preload = 100000;
+      http-port = 8989;
+      run-metrics = true;
+      run-health-check = true;
+      logger-level = "info";
+      logger-format = "simple";
+      logger-secrets = [
+        "MESSAGES_PASSWORD"
+        "EXECUTION_NODE"
+        "CONSENSUS_NODE"
+      ];
+      dry-run = false;
+    };
+  };
+}

machines/modules/lido-withdrawals-automation.nix

@@ -0,0 +1,17 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.lido-withdrawals-automation
+  ];
+
+  services.lido-withdrawals-automation = {
+    enable = true;
+    args = {
+      operator-id = "";
+      password = "";
+      percentage = 10;
+      output-folder = "/ethereum/lido/withdrawal-automation";
+      overwrite = "always";
+    };
+  };
+}

machines/modules/mcl-disko.nix

@@ -0,0 +1,8 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.mcl-disko
+  ];
+
+  #TODO: Figure out the arguments for this service and enable it
+}

machines/modules/mcl-host-info.nix

@@ -0,0 +1,12 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.mcl-host-info
+  ];
+
+  mcl.host-info = {
+    type = "server";
+    isDebugVM = true;
+    configPath = ./.;
+  };
+}

machines/modules/mcl-secrets.nix

@@ -0,0 +1,8 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.mcl-secrets
+  ];
+
+  #TODO: Figure out the arguments for this service and enable it
+}

machines/modules/pharos.nix

@@ -0,0 +1,11 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.pharos
+  ];
+
+  services.pharos = {
+    enable = true;
+    network = "testnet";
+  };
+}

machines/modules/pyroscope.nix

@@ -0,0 +1,10 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.pyroscope
+  ];
+
+  services.pyroscope = {
+    enable = true;
+  };
+}

machines/modules/random-alerts.nix

@@ -0,0 +1,8 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.random-alerts
+  ];
+
+  #TODO: Figure out the arguments for this service and enable it
+}

modules/default.nix

@@ -9,5 +9,6 @@
     ./secrets.nix
     ./mcl-disko
     ./pharos
+    ./healthcheck
   ];
 }