diff --git a/.gitignore b/.gitignore index a12064f..893d2bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode build/* -GeoLite2City \ No newline at end of file +GeoLite2City +secrets diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index e67fe88..61b98a1 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -49,15 +49,13 @@ services: - 9281 volumes: - "./mirror-sync-scheduler/storage:/storage" + - "./mirror-sync-scheduler/error-logs:/mirror/error-logs" - "./mirror-sync-scheduler/configs:/mirror/configs:ro" - "./mirror-sync-scheduler/scripts:/mirror/scripts:ro" + - "./mirror-sync-scheduler/secrets:/mirror/secrets:ro" networks: - mirror - depends_on: - - log-server - stdin_open: true tty: true - # --- Torrent handler --- # torrent-handler: # build: ./mirror-torrent-handler diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 3e5f490..21107ed 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -69,14 +69,17 @@ services: - configs/sync-scheduler.env volumes: - "/storage:/storage" + - "./sync-error-logs:/mirror/error-logs" - "./configs:/mirror/configs:ro" - "./scripts:/mirror/scripts:ro" + - "./secrets:/mirror/secrets:ro" networks: - mirror - depends_on: - - log-server - stdin_open: true tty: true + logging: + options: + max-size: 1g + max-file: 5 # --- Torrent handler --- # torrent-handler: diff --git a/mirror-sync-scheduler/.gitignore b/mirror-sync-scheduler/.gitignore index 83ae538..f407a01 100644 --- a/mirror-sync-scheduler/.gitignore +++ b/mirror-sync-scheduler/.gitignore @@ -2,3 +2,4 @@ build *.secret .idea +secrets diff --git a/mirror-sync-scheduler/Dockerfile b/mirror-sync-scheduler/Dockerfile index 99008de..ef96d08 100644 --- a/mirror-sync-scheduler/Dockerfile +++ b/mirror-sync-scheduler/Dockerfile @@ -14,7 +14,7 @@ RUN cmake --build build --target all # Run FROM ubuntu:24.04 -RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync +RUN apt update && apt install -y python3 curl wget libzmq3-dev rsync zsh WORKDIR /mirror COPY --from=builder /mirror/build/src/sync_scheduler . RUN chmod 744 sync_scheduler diff --git a/mirror-sync-scheduler/configs/mirrors.json b/mirror-sync-scheduler/configs/mirrors.json index a079c17..0f765ac 100644 --- a/mirror-sync-scheduler/configs/mirrors.json +++ b/mirror-sync-scheduler/configs/mirrors.json @@ -572,10 +572,8 @@ "name": "Raspbian", "page": "Distributions", "script": { - "env": { - "mirror": "mirror.umd.edu" - }, - "command": "python3 raspbmirror.py --tmpdir /storage/raspbian-tmp/ --sourcepool /storage/debian/pool http://${mirror}/raspbian http://${mirror}/raspbian http://snapshot.raspbian.org/hashpool", + "command": "python3", + "arguments": ["raspbmirror.py", "--tmpdir", "/storage/raspbian-tmp/", "--sourcepool", "/storage/debian/pool", "http://mirror.umd.edu/raspbian", "http://mirror.umd.edu/raspbian", "http://snapshot.raspbian.org/hashpool"], "syncs_per_day": 3 }, "official": false, diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp index 8c8d814..1fbdb0d 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/JobManager.hpp @@ -38,7 +38,7 @@ class JobManager public: // Methods auto start_job( const std::string& jobName, - std::string command, + std::vector command, const std::filesystem::path& passwordFile ) -> bool; @@ -53,13 +53,19 @@ class JobManager auto kill_all_jobs() -> void; auto reap_processes() -> std::vector<::pid_t>; auto deregister_jobs(const std::vector<::pid_t>& completedJobs) -> void; + auto write_streams_to_file(const ::pid_t processID) + -> std::pair; auto process_reaper(const std::stop_token& stopToken) -> void; private: // Static Methods - static auto get_child_process_ids(const ::pid_t processID) + static auto get_child_process_ids(const ::pid_t processID = ::getpid()) -> std::vector<::pid_t>; static auto interrupt_job(const ::pid_t processID) -> void; static auto kill_job(const ::pid_t processID) -> void; + static auto write_stream_to_file( + const std::string& logfileName, + const int pipeFileDescriptor + ) -> bool; private: // Members std::map<::pid_t, SyncJob> m_ActiveJobs; diff --git a/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp b/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp index 3e0fae5..06916cb 100644 --- a/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp +++ b/mirror-sync-scheduler/include/mirror/sync_scheduler/SyncDetails.hpp @@ -40,7 +40,7 @@ class SyncDetails } [[nodiscard]] - auto get_commands() const -> std::vector + auto get_commands() const -> std::vector> { return m_Commands; } @@ -48,18 +48,20 @@ class SyncDetails private: // Methods [[nodiscard]] static auto compose_rsync_commands(const nlohmann::json& rsyncConfig) + -> std::vector>; + + static auto handle_rsync_options_array(const nlohmann::json& optionsArray) -> std::vector; auto handle_sync_config(const nlohmann::json& project) -> void; auto handle_rsync_config(const nlohmann::json& project) -> void; - auto handle_script_config(const nlohmann::json& project) -> void; private: // Members - std::size_t m_SyncsPerDay; - std::optional m_PasswordFile; - std::vector m_Commands; + std::size_t m_SyncsPerDay; + std::optional m_PasswordFile; + std::vector> m_Commands; }; struct static_project_exception : std::runtime_error diff --git a/mirror-sync-scheduler/src/JobManager.cpp b/mirror-sync-scheduler/src/JobManager.cpp index d6049ca..f8a169c 100644 --- a/mirror-sync-scheduler/src/JobManager.cpp +++ b/mirror-sync-scheduler/src/JobManager.cpp @@ -8,6 +8,7 @@ #include // System Includes +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -26,9 +28,12 @@ #include #include #include +#include +#include #include #include #include +#include #include // Third Party Includes @@ -70,20 +75,26 @@ auto JobManager::process_reaper(const std::stop_token& stopToken) -> void return; } - auto completedJobs = reap_processes(); + auto completedJobs = this->reap_processes(); this->deregister_jobs(completedJobs); completedJobs.clear(); } } -auto JobManager::get_child_process_ids(const ::pid_t processID = ::getpid()) +auto JobManager::get_child_process_ids(const ::pid_t processID) -> std::vector<::pid_t> { - static const std::filesystem::path taskDirectory + const std::filesystem::path taskDirectory = std::filesystem::absolute(std::format("/proc/{}/task/", processID)); std::vector<::pid_t> childProcesses = {}; + + spdlog::trace( + "Gathering child process ids for process with pid {}", + processID + ); + for (const auto& taskEntry : std::filesystem::directory_iterator(taskDirectory)) { @@ -118,8 +129,6 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> { std::string errorMessage(BUFSIZ, '\0'); - static const ::pid_t syncSchedulerProcessID = ::getpid(); - std::vector<::pid_t> completedJobs; completedJobs.reserve(m_ActiveJobs.size()); @@ -133,7 +142,7 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> int status = 0; const bool isKnownJob = m_ActiveJobs.contains(childProcessID); - constexpr auto JOB_TIMEOUT = std::chrono::hours(6); + constexpr auto JOB_TIMEOUT = std::chrono::hours(3); std::chrono::hours syncDuration; // NOLINTNEXTLINE(misc-include-cleaner) @@ -167,8 +176,7 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> spdlog::warn( "Project {} has been syncing for at least {} hour{}. " - "Process " - "may be hanging, attempting to send SIGTERM. (pid: {})", + "Process may be hanging, attempting to send SIGTERM. (pid: {})", m_ActiveJobs.at(childProcessID).jobName, JOB_TIMEOUT.count(), // if not one hour, plural @@ -188,33 +196,39 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> if (isKnownJob && exitStatus == EXIT_SUCCESS) { spdlog::info( - "Project {} successfully synced! (pid: {})", - m_ActiveJobs.at(childProcessID).jobName, - childProcessID + "Project {} successfully synced!", + m_ActiveJobs.at(childProcessID).jobName ); completedJobs.emplace_back(childProcessID); } else if (isKnownJob && exitStatus != EXIT_SUCCESS) { + const auto [stdoutLogfile, stderrLogfile] + = this->write_streams_to_file(childProcessID); + spdlog::warn( - "Project {} failed to sync! Exit code: {} (pid: {})", + "Project {} failed to sync! Attempted to write sync output " + "to {} and {}. Exit code: {}", m_ActiveJobs.at(childProcessID).jobName, - exitStatus, - childProcessID + stdoutLogfile, + stderrLogfile, + exitStatus ); completedJobs.emplace_back(childProcessID); } else if (!isKnownJob && exitStatus == EXIT_SUCCESS) { spdlog::info( - "Reaped successful unregistered child process with pid {}", + "Reaped successful unregistered child process with pid " + "{}", childProcessID ); } else { spdlog::warn( - "Reaped unsuccessful unregistered child process with pid " + "Reaped unsuccessful unregistered child process with " + "pid " "{}", childProcessID ); @@ -225,16 +239,98 @@ auto JobManager::reap_processes() -> std::vector<::pid_t> return completedJobs; } +auto JobManager::write_streams_to_file(const ::pid_t processID) + -> std::pair +{ + static std::random_device randomDevice; + static std::mt19937 randomGenerator(randomDevice()); + static std::uniform_int_distribution distribution(1, 10000); + + const auto logNumber = distribution(randomGenerator); + + const auto stdoutLogFileName = std::format( + "{}-{}-stdout.log", + m_ActiveJobs.at(processID).jobName, + logNumber + ); + + const auto stderrLogFileName = std::format( + "{}-{}-stderr.log", + m_ActiveJobs.at(processID).jobName, + logNumber + ); + + const bool stdoutSuccess = write_stream_to_file( + stdoutLogFileName, + m_ActiveJobs.at(processID).stdoutPipe + ); + + const bool stderrSuccess = write_stream_to_file( + stderrLogFileName, + m_ActiveJobs.at(processID).stderrPipe + ); + + return { (stdoutSuccess ? stdoutLogFileName : ""), + (stderrSuccess ? stderrLogFileName : "") }; +} + +auto JobManager::write_stream_to_file( + const std::string& logfileName, + const int pipeFileDescriptor +) -> bool +{ + const auto errorLogPath = std::filesystem::relative("error-logs"); + std::ofstream logfileStream(errorLogPath / logfileName); + + if (!logfileStream.good()) + { + spdlog::error("Failed to open log file {}!", logfileName); + + return false; + } + + std::string streamContents; + streamContents.resize(BUFSIZ); + + std::ptrdiff_t bytesRead = 1; + + while (bytesRead != 0 && bytesRead != -1) + { + bytesRead = ::read( + pipeFileDescriptor, + streamContents.data(), + streamContents.size() + ); + + // The buffer was filled + if (bytesRead == streamContents.size()) + { + logfileStream << streamContents; + } + else // Fewer characters were read than the buffer could fit + { + streamContents.resize(bytesRead); + logfileStream << streamContents; + } + } + + return true; +} + // NOLINTNEXTLINE(*-no-recursion) auto JobManager::interrupt_job(const ::pid_t processID) -> void { - // Interrupt child processes recursively. Starts with the grandest child and - // works its way back up to the direct decendant of the sync scheduler + // Interrupt child processes recursively. Starts with the grandest child + // and works its way back up to the direct descendant of the sync + // scheduler // - // Base case: process with no children. `get_child_process_ids` will be an - // empty collection meaning nothing to iterate over - for (const ::pid_t childProcessID : JobManager::get_child_process_ids()) + // Base case: process with no children. `get_child_process_ids` will be + // an empty collection meaning nothing to iterate over + + for (const ::pid_t childProcessID : + JobManager::get_child_process_ids(processID)) { + spdlog::trace("Interrupting job with pid {}", childProcessID); JobManager::interrupt_job(childProcessID); } @@ -257,29 +353,31 @@ auto JobManager::interrupt_job(const ::pid_t processID) -> void spdlog::debug("Successfully sent process {} a SIGTERM", processID); - constexpr auto SIGTERM_TIMEOUT = std::chrono::seconds(30); - const auto start = std::chrono::system_clock::now(); - auto now = start; + // TODO: Handle asynchronously instead of commenting out. Also find some way + // to account for process possibly being reaped by its parent + /* constexpr auto SIGTERM_TIMEOUT = std::chrono::seconds(30); + const auto start = std::chrono::system_clock::now(); + auto now = start; - while ((now - start) < SIGTERM_TIMEOUT) - { - const int waitReturn = ::waitpid(processID, nullptr, WNOHANG); - - if (waitReturn == processID) + while ((now - start) < SIGTERM_TIMEOUT) { - spdlog::trace("Process {} successfully reaped", processID); - return; - } + const int waitReturn = ::waitpid(processID, nullptr, WNOHANG); - constexpr auto CHECK_INTERVAL = std::chrono::milliseconds(100); - std::this_thread::sleep_for(CHECK_INTERVAL); + if (waitReturn == processID) + { + spdlog::trace("Process {} successfully reaped", processID); + return; + } - now = std::chrono::system_clock::now(); - } + constexpr auto CHECK_INTERVAL = std::chrono::milliseconds(100); + std::this_thread::sleep_for(CHECK_INTERVAL); - spdlog::error("Failed to terminate process {} with SIGTERM", processID); + now = std::chrono::system_clock::now(); + } - JobManager::kill_job(processID); + spdlog::error("Failed to terminate process {} with SIGTERM", processID); + + JobManager::kill_job(processID); */ } auto JobManager::job_is_running(const std::string& jobName) -> bool @@ -313,6 +411,8 @@ auto JobManager::kill_job(const ::pid_t processID) -> void // NOLINTNEXTLINE(*-include-cleaner) ::strerror_r(errno, errorMessage.data(), errorMessage.size()) ); + + return; } const int waitReturn = ::waitpid(processID, nullptr, 0); @@ -377,6 +477,9 @@ auto JobManager::deregister_jobs(const std::vector<::pid_t>& completedJobs) const std::lock_guard jobLock(m_JobMutex); for (const auto& job : completedJobs) { + ::close(m_ActiveJobs.at(job).stderrPipe); + ::close(m_ActiveJobs.at(job).stdoutPipe); + m_ActiveJobs.erase(job); } } @@ -384,7 +487,7 @@ auto JobManager::deregister_jobs(const std::vector<::pid_t>& completedJobs) // NOLINTBEGIN(*-easily-swappable-parameters) auto JobManager::start_job( const std::string& jobName, - std::string command, + std::vector command, const std::filesystem::path& passwordFile ) -> bool // NOLINTEND(*-easily-swappable-parameters) @@ -409,7 +512,8 @@ auto JobManager::start_job( std::string errorMessage(BUFSIZ, '\0'); spdlog::warn( - "Failed to create pipe for child stdout while syncing project {}! " + "Failed to create pipe for child stdout while syncing project " + "{}! " "Error message: {}", jobName, // NOLINTNEXTLINE(*-include-cleaner) @@ -469,20 +573,21 @@ auto JobManager::start_job( ); } - //! ::strdup allocates memory with ::malloc. Typically this memory - //! should be ::free'd. However, argv is supposed to have the same - //! lifetime as the process it belongs to, therefore the memory should - //! never be freed and we do not need to maintain a copy of the pointer - //! to free it at a later time - // NOLINTBEGIN(*-include-cleaner) - const std::array argv { ::strdup("/bin/sh"), - ::strdup("-c"), - ::strdup(command.data()), - nullptr }; - // NOLINTEND(*-include-cleaner) + std::vector argv; + argv.resize(command.size() + 1); + + std::transform( + std::begin(command), + std::end(command), + std::begin(argv), + [](std::string& str) { return std::data(str); } + ); + + // Last item in argv has to be a nullptr; + argv.emplace_back(nullptr); spdlog::debug("Setting process group ID"); - ::setpgid(0, 0); + //::setpgid(0, 0); spdlog::trace("Calling exec"); ::execv(argv.at(0), argv.data()); @@ -491,7 +596,8 @@ auto JobManager::start_job( std::string errorMessage(BUFSIZ, '\0'); spdlog::error( - "Call to execv() failed while trying to sync {}! Error message: {}", + "Call to execv() failed while trying to sync {}! Error " + "message: {}", jobName, // NOLINTNEXTLINE(*-include-cleaner) ::strerror_r(errno, errorMessage.data(), errorMessage.size()) @@ -520,7 +626,7 @@ auto JobManager::start_job( // Close write end of the stderr pipe in the parent process ::close(stderrPipes.at(1)); - this->register_job(jobName, pid, stdoutPipes.at(1), stderrPipes.at(1)); + this->register_job(jobName, pid, stdoutPipes.at(0), stderrPipes.at(0)); return true; } } // namespace mirror::sync_scheduler diff --git a/mirror-sync-scheduler/src/SyncDetails.cpp b/mirror-sync-scheduler/src/SyncDetails.cpp index 34fed04..c649a39 100644 --- a/mirror-sync-scheduler/src/SyncDetails.cpp +++ b/mirror-sync-scheduler/src/SyncDetails.cpp @@ -17,6 +17,9 @@ // Third Party Includes #include +#include +// NOLINTNEXTLINE(misc-include-cleaner) Required to print a range in a log +#include namespace mirror::sync_scheduler { @@ -25,10 +28,12 @@ SyncDetails::SyncDetails(const nlohmann::json& project) { if (project.contains("static")) { - throw static_project_exception(std::format( - "Project '{}' uses a static sync. Project not created!", - project.value("name", "") - )); + throw static_project_exception( + std::format( + "Project '{}' uses a static sync. Project not created!", + project.value("name", "") + ) + ); } const bool isRsyncOrScript @@ -36,19 +41,21 @@ SyncDetails::SyncDetails(const nlohmann::json& project) if (!isRsyncOrScript) { - throw std::runtime_error(std::format( - "Project '{}' is missing a sync type!", - project.value("name", "") - )); + throw std::runtime_error( + std::format( + "Project '{}' is missing a sync type!", + project.value("name", "") + ) + ); } SyncDetails::handle_sync_config(project); } auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) - -> std::vector + -> std::vector> { - std::vector commands; + std::vector> commands; using namespace std::string_view_literals; constexpr auto RSYNC_EXECUTABLE = "/usr/bin/rsync"sv; @@ -58,44 +65,71 @@ auto SyncDetails::compose_rsync_commands(const nlohmann::json& rsyncConfig) const std::string src = rsyncConfig.value("src", ""); const std::string dest = rsyncConfig.value("dest", ""); - for (const std::string options : rsyncConfig.at("options")) + for (const nlohmann::json& options : rsyncConfig.at("options")) { if (options.empty()) { - throw std::runtime_error(std::format( - "Project {} contains invalid rsync config options", - rsyncConfig.value("name", "") - )); + throw std::runtime_error( + std::format( + "Project {} contains invalid rsync config options", + rsyncConfig.value("name", "") + ) + ); + } + + spdlog::trace("Options: {}", options.dump()); + + if (options.is_array()) + { + commands.emplace_back( + SyncDetails::handle_rsync_options_array(options) + ); + } + else if (options.is_string()) + { + if (commands.empty()) + { + commands.resize(1); + + commands.front().emplace_back("/usr/bin/rsync"); + } + + commands.front().emplace_back(options); } + } + for (auto& command : commands) + { if (!user.empty()) { - commands.emplace_back(std::format( - "{} {} {}@{}::{} {}", - RSYNC_EXECUTABLE, - options, - user, - host, - src, - dest - )); + command.emplace_back(std::format("{}@{}::{}", user, host, src)); } else { - commands.emplace_back(std::format( - "{} {} {}::{} {}", - RSYNC_EXECUTABLE, - options, - host, - src, - dest - )); + command.emplace_back(std::format("{}::{}", host, src)); } + + command.emplace_back(dest); + + spdlog::trace("Sync Command: {{ {} }}", fmt::join(command, ", ")); } return commands; } +auto SyncDetails::handle_rsync_options_array(const nlohmann::json& optionsArray) + -> std::vector +{ + std::vector toReturn = { "/usr/bin/rsync" }; + + for (const std::string option : optionsArray) + { + toReturn.emplace_back(option); + } + + return toReturn; +} + auto SyncDetails::handle_sync_config(const nlohmann::json& project) -> void { if (project.contains("rsync")) @@ -128,17 +162,21 @@ auto SyncDetails::handle_rsync_config(const nlohmann::json& project) -> void auto SyncDetails::handle_script_config(const nlohmann::json& project) -> void { - m_SyncsPerDay = project.at("script").at("syncs_per_day").get(); + const auto& scriptBlock = project.at("script"); - std::string command = project.at("script").at("command"); - const std::vector arguments - = project.at("script").value("arguments", std::vector {}); + m_SyncsPerDay = scriptBlock.at("syncs_per_day").get(); - for (const std::string& arg : arguments) + std::vector toReturn = { "/usr/bin/bash", "-c" }; + + toReturn.emplace_back(scriptBlock.at("command")); + + for (const std::string arg : scriptBlock.at("arguments")) { - command = std::format("{} {}", command, arg); + toReturn.back() = std::format("{} {}", toReturn.back(), arg); } - m_Commands.emplace_back(command); + spdlog::trace("Sync Command: {{ {} }}", fmt::join(toReturn, ", ")); + + m_Commands.emplace_back(toReturn); } } // namespace mirror::sync_scheduler diff --git a/mirror-sync-scheduler/src/SyncScheduler.cpp b/mirror-sync-scheduler/src/SyncScheduler.cpp index e6ed6d7..bde16a6 100644 --- a/mirror-sync-scheduler/src/SyncScheduler.cpp +++ b/mirror-sync-scheduler/src/SyncScheduler.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -40,10 +41,12 @@ namespace mirror::sync_scheduler { SyncScheduler::SyncScheduler() try // Function try block my beloved - : m_ProjectCatalogue(SyncScheduler::generate_project_catalogue( - SyncScheduler::load_json_config("configs/mirrors.json") - .value("mirrors", nlohmann::json()) - )), + : m_ProjectCatalogue( + SyncScheduler::generate_project_catalogue( + SyncScheduler::load_json_config("configs/mirrors.json") + .value("mirrors", nlohmann::json()) + ) + ), m_Schedule(m_ProjectCatalogue), m_DryRun(false) { @@ -90,12 +93,14 @@ auto SyncScheduler::load_json_config(const std::filesystem::path& file) { std::string errorMessage(BUFSIZ, '\0'); - throw std::runtime_error(std::format( - "Failed to load config file {}! OS Error: {}", - file.filename().string(), - // NOLINTNEXTLINE(*-include-cleaner) - ::strerror_r(errno, errorMessage.data(), errorMessage.size()) - )); + throw std::runtime_error( + std::format( + "Failed to load config file {}! OS Error: {}", + file.filename().string(), + // NOLINTNEXTLINE(*-include-cleaner) + ::strerror_r(errno, errorMessage.data(), errorMessage.size()) + ) + ); } return nlohmann::json::parse(mirrorsConfigFile); @@ -120,11 +125,15 @@ auto SyncScheduler::generate_project_catalogue(const nlohmann::json& mirrors) } catch (static_project_exception& spe) { - spdlog::trace(spe.what()); + spdlog::trace("{}: {}", name, spe.what()); } catch (std::runtime_error& re) { - spdlog::error(re.what()); + spdlog::error("{}: {}", name, re.what()); + } + catch (std::exception& e) + { + spdlog::error("{}: {}", name, e.what()); } } @@ -139,7 +148,7 @@ auto SyncScheduler::start_sync(const std::string& projectName) -> bool return true; } - spdlog::info("Attempting sync for {}", projectName); + spdlog::debug("Attempting sync for {}", projectName); const auto& syncDetails = m_ProjectCatalogue.at(projectName); bool startSuccessful = false; @@ -175,6 +184,8 @@ auto SyncScheduler::start_sync(const std::string& projectName) -> bool auto SyncScheduler::manual_sync_loop() -> void { + using namespace std::string_view_literals; + zmq::context_t socketContext {}; zmq::socket_t socket { socketContext, zmq::socket_type::rep }; @@ -195,38 +206,81 @@ auto SyncScheduler::manual_sync_loop() -> void [[maybe_unused]] auto result = socket.recv(syncRequest, zmq::recv_flags::none); - const std::string projectName = syncRequest.to_string(); + const std::string requestedProjectName = syncRequest.to_string(); - spdlog::info("Manual sync requested for {}", projectName); + spdlog::info("Manual sync requested for {}", requestedProjectName); - if (m_ProjectCatalogue.contains(projectName)) + if (m_ProjectCatalogue.contains(requestedProjectName)) { - if (SyncScheduler::start_sync(projectName)) + if (SyncScheduler::start_sync(requestedProjectName)) { socket.send( zmq::message_t( - std::format("SUCCESS: started sync for {}", projectName) + std::format( + "SUCCESS: started sync for {}", + requestedProjectName + ) ), zmq::send_flags::none ); continue; } - spdlog::error("Manual sync for {} failed", projectName); + spdlog::error("Manual sync for {} failed", requestedProjectName); + socket.send( + zmq::message_t( + std::format( + "FAILURE: Failed to start sync for {}", + requestedProjectName + ) + ), + zmq::send_flags::none + ); + continue; + } + else if (requestedProjectName == "all_projects") + { + bool allSyncsStarted = true; + + for (const auto& [projectName, _] : m_ProjectCatalogue) + { + const bool syncStarted = SyncScheduler::start_sync(projectName); + allSyncsStarted = allSyncsStarted && syncStarted; + + if (!syncStarted) + { + spdlog::error( + "Failed to start manual sync for {}", + projectName + ); + } + } + + if (allSyncsStarted) + { + socket.send( + zmq::message_t("SUCCESS: started syncing all projects"sv), + zmq::send_flags::none + ); + continue; + } + socket.send( - zmq::message_t(std::format( - "FAILURE: Failed to start sync for {}", - projectName - )), + zmq::message_t( + "FAILURE: Failed to start sync for some projects"sv + ), zmq::send_flags::none ); continue; } - spdlog::error("Project {} not found!", projectName); + spdlog::error("Project {} not found!", requestedProjectName); socket.send( zmq::message_t( - std::format("FAILURE: Project {} not found!", projectName) + std::format( + "FAILURE: Project {} not found!", + requestedProjectName + ) ), zmq::send_flags::none ); diff --git a/scripts/kicad/kicad_sync.py b/scripts/kicad/kicad_sync.py index 6916b2c..46d3f78 100644 --- a/scripts/kicad/kicad_sync.py +++ b/scripts/kicad/kicad_sync.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import subprocess from time import sleep import os @@ -8,38 +10,46 @@ # Path of download root (ex. "/storage/kicad/") DOWNLOAD_PREFIX = "/storage/kicad/" + def runCommandWithOutput(args: list[str]) -> str: return subprocess.run(args, stdout=subprocess.PIPE).stdout.decode() + def runCommand(args: list[str]) -> str: subprocess.run(args) + def isFile(path: str): return not (path.endswith("/") or path.endswith("index.html") or path.endswith("list.js") or path.endswith("favicon.ico")) + def isValue(line: str): return not line.startswith("<") + def getNextMarker(xml: str): - lines: list[str] = xml.replace("", "\n").replace("", "\n").splitlines() + lines: list[str] = xml.replace("", "\n").replace( + "", "\n").splitlines() markers: list[str] = list(filter(lambda x: isValue(x), lines)) - if(len(markers) > 0): + if (len(markers) > 0): return markers[0] return "" + def getFilePaths(bucket_url: str) -> list[str]: paths: list[str] = [] suffix: str = "" i: int = 0 - while(True): + while (True): print("----- Fetching block {} of index... -----".format(i)) full_url: str = "{}{}".format(bucket_url, suffix) xml: str = runCommandWithOutput(["curl", "-s", full_url]) - lines: list[str] = xml.replace("", "\n").replace("", "\n").splitlines() + lines: list[str] = xml.replace("", "\n").replace( + "", "\n").splitlines() paths.extend(list(filter(lambda x: isValue(x) and isFile(x), lines))) next_marker: str = getNextMarker(xml) - if(next_marker == ""): + if (next_marker == ""): print("----- Done fetching index. -----") break suffix = "?marker={}".format(next_marker) @@ -47,13 +57,15 @@ def getFilePaths(bucket_url: str) -> list[str]: sleep(0.25) return paths + def downloadFile(bucket_url: str, dl_prefix: str, path: str): full_url: str = bucket_url + "/" + path - if(os.path.exists("{}{}".format(dl_prefix, path))): + if (os.path.exists("{}{}".format(dl_prefix, path))): print("Skipping this file (already exists).") else: runCommand(["wget", "-c", "-nH", "-x", "-P", dl_prefix, full_url]) + def main(): print("----- Fetching index... -----") paths: list[str] = getFilePaths(BUCKET_URL) @@ -65,9 +77,10 @@ def main(): downloadFile(BUCKET_URL, DOWNLOAD_PREFIX, path) i += 1 -if(__name__ == "__main__"): + +if (__name__ == "__main__"): try: main() - except(KeyboardInterrupt): + except (KeyboardInterrupt): print("CTRL-C") pass