Add support in config for including files from dir

Author: halfgaar

CMakeLists.txt

@@ -76,6 +76,7 @@ add_executable(flashmq
     acksender.h
     bridgeconfig.h
     dnsresolver.h
+    globber.h
 
 
     mainapp.cpp
@@ -130,6 +131,7 @@ add_executable(flashmq
     bridgeconfig.cpp
     dnsresolver.cpp
     bridgeinfodb.h bridgeinfodb.cpp
+    globber.cpp
 
     )
 

FlashMQTests/FlashMQTests.pro

@@ -65,6 +65,7 @@ SOURCES +=  tst_maintests.cpp \
     ../bridgeconfig.cpp \
     ../dnsresolver.cpp \
     ../bridgeinfodb.cpp \
+    ../globber.cpp \
     conffiletemp.cpp \
     dnstests.cpp \
     flashmqtempdir.cpp \
@@ -129,6 +130,7 @@ HEADERS += \
     ../bridgeconfig.h \
     ../dnsresolver.h \
     ../bridgeinfodb.h \
+    ../globber.h \
     conffiletemp.h \
     flashmqtempdir.h \
     mainappthread.h \

configfileparser.cpp

@@ -16,12 +16,14 @@ See LICENSE for license details.
 #include <fstream>
 #include <regex>
 #include <sys/stat.h>
+#include <filesystem>
 
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 
 #include "exceptions.h"
 #include "utils.h"
+#include "globber.h"
 
 
 /**
@@ -148,6 +150,7 @@ ConfigFileParser::ConfigFileParser(const std::string &path) :
     validKeys.insert("max_outgoing_topic_alias_value");
     validKeys.insert("client_max_write_buffer_size");
     validKeys.insert("retained_messages_delivery_limit");
+    validKeys.insert("include_dir");
 
     validListenKeys.insert("port");
     validListenKeys.insert("protocol");
@@ -182,10 +185,12 @@ ConfigFileParser::ConfigFileParser(const std::string &path) :
     validBridgeKeys.insert("max_incoming_topic_aliases");
 }
 
-void ConfigFileParser::loadFile(bool test)
+std::list<std::string> ConfigFileParser::readFileRecursively(const std::string &path) const
 {
+    std::list<std::string> lines;
+
     if (path.empty())
-        return;
+        return lines;
 
     checkFileExistsAndReadable("application config file", path, 1024*1024*10);
 
@@ -198,18 +203,64 @@ void ConfigFileParser::loadFile(bool test)
         throw ConfigFileException(oss.str());
     }
 
-    std::list<std::string> lines;
+    for(std::string line; getline(infile, line ); )
+    {
+        lines.push_back(line);
+
+        if (strContains(line, "include_dir"))
+        {
+            std::smatch matches;
+
+            if (std::regex_match(line, matches, key_value_regex))
+            {
+                const std::string key = matches[1].str();
+                const std::string value = matches[2].str();
+
+                if (key == "include_dir")
+                {
+                    Logger *logger = Logger::getInstance();
+
+                    if (!std::filesystem::is_directory(value))
+                    {
+                        std::ostringstream oss; oss << "Include_dir '" << value << "' is not there or is not a directory";
+                        throw ConfigFileException(oss.str());
+                    }
+
+                    Globber globber;
+                    std::vector<std::string> files = globber.getGlob(value + "/*.conf");
 
-    const std::regex key_value_regex("^([a-zA-Z0-9_\\-]+) +([a-zA-Z0-9_\\-/.:#+]+) *([a-zA-Z0-9_\\-/.:#+]+)?$");
-    const std::regex block_regex_start("^([a-zA-Z0-9_\\-]+) *\\{$");
-    const std::regex block_regex_end("^\\}$");
+                    if (files.empty())
+                    {
+                        logger->logf(LOG_WARNING, "Including '%s' yielded 0 files.", value.c_str());
+                    }
+
+                    for(const std::string &path_from_glob : files)
+                    {
+                        std::list<std::string> newLines = readFileRecursively(path_from_glob);
+                        lines.insert(lines.cend(), newLines.begin(), newLines.end());
+                    }
+                }
+            }
+        }
+    }
+
+    return lines;
+}
+
+void ConfigFileParser::loadFile(bool test)
+{
+    if (path.empty())
+        return;
+
+    const std::list<std::string> unprocessed_lines = readFileRecursively(path);
+    std::list<std::string> lines;
 
     bool inBlock = false;
     std::ostringstream oss;
     int linenr = 0;
 
     // First parse the file and keep the valid lines.
-    for(std::string line; getline(infile, line ); )
+    for (std::string line : unprocessed_lines)
     {
         trim(line);
         linenr++;

configfileparser.h

@@ -18,6 +18,7 @@ See LICENSE for license details.
 #include <memory>
 #include <list>
 #include <limits>
+#include <regex>
 
 #include "settings.h"
 
@@ -35,6 +36,10 @@ class ConfigFileParser
     std::set<std::string> validListenKeys;
     std::set<std::string> validBridgeKeys;
 
+    const std::regex key_value_regex = std::regex("^([a-zA-Z0-9_\\-]+) +([a-zA-Z0-9_\\-/.:#+]+) *([a-zA-Z0-9_\\-/.:#+]+)?$");
+    const std::regex block_regex_start = std::regex("^([a-zA-Z0-9_\\-]+) *\\{$");
+    const std::regex block_regex_end = std::regex("^\\}$");
+
     Settings settings;
 
     bool testKeyValidity(const std::string &key, const std::string &matchKey, const std::set<std::string> &validKeys) const;
@@ -45,6 +50,7 @@ class ConfigFileParser
 public:
     ConfigFileParser(const std::string &path);
     void loadFile(bool test);
+    std::list<std::string> readFileRecursively(const std::string &path) const;
 
     const Settings &getSettings();
 };

globber.cpp

@@ -0,0 +1,35 @@
+#include "globber.h"
+
+#include <cstring>
+#include <stdexcept>
+
+GlobT::GlobT()
+{
+    std::memset(&result, 0, sizeof (glob_t));
+}
+
+GlobT::~GlobT()
+{
+    globfree(&result);
+}
+
+std::vector<std::string> Globber::getGlob(const std::string &pattern)
+{
+    GlobT result;
+
+    const int rc = glob(pattern.c_str(), 0, nullptr, &result.result);
+
+    if (!(rc == 0 || rc == GLOB_NOMATCH))
+    {
+        throw std::runtime_error("Glob failed");
+    }
+
+    std::vector<std::string> filenames;
+
+    for(size_t i = 0; i < result.result.gl_pathc; ++i)
+    {
+        filenames.push_back(std::string(result.result.gl_pathv[i]));
+    }
+
+    return filenames;
+}

globber.h

@@ -0,0 +1,26 @@
+#ifndef GLOBBER_H
+#define GLOBBER_H
+
+#include <glob.h>
+#include <string>
+#include <vector>
+
+class GlobT
+{
+    friend class Globber;
+
+    glob_t result;
+
+public:
+    GlobT();
+    ~GlobT();
+};
+
+class Globber
+{
+public:
+
+    std::vector<std::string> getGlob(const std::string &pattern);
+};
+
+#endif // GLOBBER_H

man/flashmq.conf.5

@@ -220,6 +220,11 @@ When having multiple subscribers on a shared subscription (like '$share/myshare/
 \fIsender_hash\fR. Selects a receiver deterministically based on the hash of the client ID of the sender. The selected subscriber will depend on how many subscribers there are, so if some disconnect, the distribution will change. Moreover, the selection may also change when FlashMQ cleans up empty spaces in the list of shared subscribers.
 
 Default: \fIround_robin\fR
+.TP 
+\*(T<\fBinclude_dir\fR\*(T> \fI/path/to/dir\fR 
+Load *.conf files from the specified directory, to merge with the main configuration file.
+
+An error is generated when the directory is not there. This is to protect against running incorrect configurations by accident, when the dir has been renamed, for example.
 .SH "LISTEN PARAMETERS"
 Listen parameters can only be used within \*(T<listen { }\*(T> blocks.
 .TP 

man/flashmq.conf.5.dbk5

@@ -451,6 +451,18 @@
         </listitem>
       </varlistentry>
 
+      <varlistentry xml:id="include_dir">
+        <term><option>include_dir</option> <replaceable>/path/to/dir</replaceable></term>
+        <listitem>
+          <para>
+            Load *.conf files from the specified directory, to merge with the main configuration file.
+          </para>
+          <para>
+            An error is generated when the directory is not there. This is to protect against running incorrect configurations by accident, when the dir has been renamed, for example.
+          </para>
+        </listitem>
+      </varlistentry>
+
     </variablelist>
   </refsect1>
 

man/flashmq.conf.5.html

@@ -506,6 +506,18 @@
         </dd>
 
 
+      
+        <dt id="include_dir"><code class="option">include_dir</code> <code class="replaceable">/path/to/dir</code><a class="hash-anchor" href="#include_dir">#</a></dt>
+        <dd>
+          <p>
+            Load *.conf files from the specified directory, to merge with the main configuration file.
+          </p>
+          <p>
+            An error is generated when the directory is not there. This is to protect against running incorrect configurations by accident, when the dir has been renamed, for example.
+          </p>
+        </dd>
+      
+
     </dl>
   </section><section class="refsect1">
     <header><h2>Listen parameters</h2></header>