Sync scheduler fixes

.gitignore

@@ -1,3 +1,4 @@
 .vscode
 build/*
-GeoLite2City
+GeoLite2City
+secrets

docker-compose-dev.yaml

@@ -49,15 +49,13 @@ services:
       - 9281
     volumes:
       - "./mirror-sync-scheduler/storage:/storage"
+      - "./mirror-sync-scheduler/error-logs:/mirror/error-logs"
       - "./mirror-sync-scheduler/configs:/mirror/configs:ro"
       - "./mirror-sync-scheduler/scripts:/mirror/scripts:ro"
+      - "./mirror-sync-scheduler/secrets:/mirror/secrets:ro"
     networks:
       - mirror
-    depends_on:
-      - log-server
-    stdin_open: true
     tty: true
-
   # --- Torrent handler ---
   #  torrent-handler:
   #    build: ./mirror-torrent-handler

docker-compose-prod.yaml

@@ -69,14 +69,17 @@ services:
       - configs/sync-scheduler.env
     volumes:
       - "/storage:/storage"
+      - "./sync-error-logs:/mirror/error-logs"
       - "./configs:/mirror/configs:ro"
       - "./scripts:/mirror/scripts:ro"
+      - "./secrets:/mirror/secrets:ro"
     networks:
       - mirror
-    depends_on:
-      - log-server
-    stdin_open: true
     tty: true
+    logging:
+      options:
+        max-size: 1g
+        max-file: 5
 
   # --- Torrent handler ---
   # torrent-handler:

mirror-sync-scheduler/.gitignore

@@ -2,3 +2,4 @@
 build
 *.secret
 .idea
+secrets

mirror-sync-scheduler/Dockerfile

@@ -14,7 +14,7 @@ RUN cmake --build build --target all
 
 # Run
 FROM ubuntu:24.04
-RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync
+RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync zsh
 WORKDIR /mirror
 COPY --from=builder /mirror/build/src/sync_scheduler .
 RUN chmod 744 sync_scheduler

mirror-sync-scheduler/configs/mirrors.json

@@ -572,10 +572,8 @@
         "name": "Raspbian",
         "page": "Distributions",
         "script": {
-          "env": {
-            "mirror": "mirror.umd.edu"
-          },
-          "command": "python3 raspbmirror.py --tmpdir /storage/raspbian-tmp/ --sourcepool /storage/debian/pool http://${mirror}/raspbian http://${mirror}/raspbian http://snapshot.raspbian.org/hashpool",
+          "command": "python3",
+          "arguments": ["raspbmirror.py", "--tmpdir", "/storage/raspbian-tmp/", "--sourcepool", "/storage/debian/pool", "http://mirror.umd.edu/raspbian", "http://mirror.umd.edu/raspbian", "http://snapshot.raspbian.org/hashpool"],
           "syncs_per_day": 3
         },
         "official": false,

mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp

@@ -38,7 +38,7 @@ class JobManager
   public: // Methods
     auto start_job(
         const std::string&           jobName,
-        std::string                  command,
+        std::vector<std::string>     command,
         const std::filesystem::path& passwordFile
     ) -> bool;
 
@@ -53,13 +53,19 @@ class JobManager
     auto kill_all_jobs() -> void;
     auto reap_processes() -> std::vector<::pid_t>;
     auto deregister_jobs(const std::vector<::pid_t>& completedJobs) -> void;
+    auto write_streams_to_file(const ::pid_t processID)
+        -> std::pair<std::string, std::string>;
     auto process_reaper(const std::stop_token& stopToken) -> void;
 
   private: // Static Methods
-    static auto get_child_process_ids(const ::pid_t processID)
+    static auto get_child_process_ids(const ::pid_t processID = ::getpid())
         -> std::vector<::pid_t>;
     static auto interrupt_job(const ::pid_t processID) -> void;
     static auto kill_job(const ::pid_t processID) -> void;
+    static auto write_stream_to_file(
+        const std::string& logfileName,
+        const int          pipeFileDescriptor
+    ) -> bool;
 
   private: // Members
     std::map<::pid_t, SyncJob> m_ActiveJobs;

mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp

@@ -40,26 +40,28 @@ class SyncDetails
     }
 
     [[nodiscard]]
-    auto get_commands() const -> std::vector<std::string>
+    auto get_commands() const -> std::vector<std::vector<std::string>>
     {
         return m_Commands;
     }
 
   private: // Methods
     [[nodiscard]]
     static auto compose_rsync_commands(const nlohmann::json& rsyncConfig)
+        -> std::vector<std::vector<std::string>>;
+
+    static auto handle_rsync_options_array(const nlohmann::json& optionsArray)
         -> std::vector<std::string>;
 
     auto handle_sync_config(const nlohmann::json& project) -> void;
 
     auto handle_rsync_config(const nlohmann::json& project) -> void;
-
     auto handle_script_config(const nlohmann::json& project) -> void;
 
   private: // Members
-    std::size_t                          m_SyncsPerDay;
-    std::optional<std::filesystem::path> m_PasswordFile;
-    std::vector<std::string>             m_Commands;
+    std::size_t                           m_SyncsPerDay;
+    std::optional<std::filesystem::path>  m_PasswordFile;
+    std::vector<std::vector<std::string>> m_Commands;
 };
 
 struct static_project_exception : std::runtime_error