Skip to content

Commit c1beaba

Browse files
kenhyskenhys
committed
system_config: support built-in config files
Currently @include directive supports to reuse configuration files, but no way to apply specific configuration files by default (without specifying to). In the previous versions, if you want to manage multiple configuration files, you must use @include directive explicitly. In this commit, add an option to specify the directory (e.g. /etc/fluent/conf.d) which stores additional configuration files. If there are such files under specified directory, they are loaded by default without @include directive. If you want to disable this feature, set empty string for config_include_dir "". Signed-off-by: Kentaro Hayashi <[email protected]>
1 parent 510b891 commit c1beaba

File tree

5 files changed

+231
-1
lines changed

5 files changed

+231
-1
lines changed

‎lib/fluent/env.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
module Fluent
2323
DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
24+
DEFAULT_CONFIG_INCLUDE_DIR = ENV["FLUENT_CONF_INCLUDE_DIR"] || '/etc/fluent/conf.d'
2425
DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
2526
DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
2627
DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'

‎lib/fluent/supervisor.rb‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
require 'fileutils'
1818
require 'open3'
1919
require 'pathname'
20+
require 'find'
2021

2122
require 'fluent/config'
2223
require 'fluent/counter'
@@ -806,6 +807,10 @@ def configure(supervisor: false)
806807

807808
$log.info :supervisor, 'parsing config file is succeeded', path: @config_path
808809

810+
build_additional_configuration do |additional_conf|
811+
@conf += additional_conf if additional_conf
812+
end
813+
809814
@libs.each do |lib|
810815
require lib
811816
end
@@ -1090,6 +1095,10 @@ def reload_config
10901095
type: @config_file_type,
10911096
)
10921097

1098+
build_additional_configuration do |additional_conf|
1099+
@conf += additional_conf if additional_conf
1100+
end
1101+
10931102
Fluent::VariableStore.try_to_reset do
10941103
Fluent::Engine.reload_config(conf)
10951104
end
@@ -1196,6 +1205,28 @@ def build_system_config(conf)
11961205
system_config
11971206
end
11981207

1208+
def build_additional_configuration
1209+
if @system_config.config_include_dir&.empty?
1210+
$log.info :supervisor, 'default configuration include directory was disabled'
1211+
return
1212+
end
1213+
begin
1214+
Find.find(@system_config.config_include_dir) do |path|
1215+
next if File.directory?(path)
1216+
next unless [".conf", ".yaml", ".yml"].include?(File.extname(path))
1217+
# NOTE: both types of normal config (.conf) and YAML will be loaded.
1218+
# Thus, it does not care whether @config_path is .conf or .yml.
1219+
$log.info :supervisor, 'loading additional configuration file', path: path
1220+
yield Fluent::Config.build(config_path: path,
1221+
encoding: @conf_encoding,
1222+
use_v1_config: @use_v1_config,
1223+
type: :guess)
1224+
end
1225+
rescue Errno::ENOENT
1226+
$log.warn :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir
1227+
end
1228+
end
1229+
11991230
RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
12001231

12011232
def build_spawn_command

‎lib/fluent/system_config.rb‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
require 'fluent/configurable'
1818
require 'fluent/config/element'
19+
require 'fluent/env'
1920

2021
module Fluent
2122
class SystemConfig
@@ -28,7 +29,8 @@ class SystemConfig
2829
:without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
2930
:file_permission, :dir_permission, :counter_server, :counter_client,
3031
:strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
31-
:metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer
32+
:metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,
33+
:config_include_dir
3234
]
3335

3436
config_param :workers, :integer, default: 1
@@ -58,6 +60,7 @@ class SystemConfig
5860
config_param :dir_permission, default: nil do |v|
5961
v.to_i(8)
6062
end
63+
config_param :config_include_dir, default: Fluent::DEFAULT_CONFIG_INCLUDE_DIR
6164
config_section :log, required: false, init: true, multi: false do
6265
config_param :path, :string, default: nil
6366
config_param :format, :enum, list: [:text, :json], default: :text

‎test/command/test_fluentd.rb‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1536,4 +1536,76 @@ def send_end(port)
15361536
end_thread&.kill
15371537
end
15381538
end
1539+
1540+
def create_config_include_dir_configuration(config_path, config_dir, yaml_format = false)
1541+
if yaml_format
1542+
conf = <<CONF
1543+
system:
1544+
config_include_dir: "#{config_dir}"
1545+
CONF
1546+
else
1547+
conf = <<CONF
1548+
<system>
1549+
config_include_dir #{config_dir}
1550+
</system>
1551+
CONF
1552+
end
1553+
create_conf_file(config_path, conf)
1554+
end
1555+
1556+
sub_test_case "test additional configuration directory" do
1557+
setup do
1558+
FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d"))
1559+
end
1560+
1561+
test "disable additional configuration directory" do
1562+
conf_path = create_config_include_dir_configuration("disabled_config_include_dir.conf", "")
1563+
assert_log_matches(create_cmdline(conf_path),
1564+
"[info]: default configuration include directory was disabled")
1565+
end
1566+
1567+
test "inaccessible include directory error" do
1568+
conf_path = create_config_include_dir_configuration("inaccessible_include.conf", "/nonexistent")
1569+
assert_log_matches(create_cmdline(conf_path),
1570+
"[warn]: inaccessible include directory was specified")
1571+
end
1572+
1573+
data("include additional configuration with relative conf.d" => [{"relative_path" => true}],
1574+
"include additional configuration with full-path conf.d" => [{"relative_path" => false}])
1575+
test "additional configuration file (conf.d/child.conf) was loaded" do |(option)|
1576+
conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d"
1577+
conf_path = create_config_include_dir_configuration("parent.conf", conf_dir)
1578+
create_conf_file('conf.d/child.conf', "")
1579+
assert_log_matches(create_cmdline(conf_path),
1580+
"[info]: loading additional configuration file path=\"#{conf_dir}/child.conf\"")
1581+
end
1582+
end
1583+
1584+
sub_test_case "test additional configuration directory (YAML)" do
1585+
setup do
1586+
FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d"))
1587+
end
1588+
1589+
test "disable additional configuration directory" do
1590+
conf_path = create_config_include_dir_configuration("disabled_config_include_dir.yml", "", true)
1591+
assert_log_matches(create_cmdline(conf_path),
1592+
"[info]: default configuration include directory was disabled")
1593+
end
1594+
1595+
test "inaccessible include directory error" do
1596+
conf_path = create_config_include_dir_configuration("inaccessible_include.yml", "/nonexistent", true)
1597+
assert_log_matches(create_cmdline(conf_path),
1598+
"[warn]: inaccessible include directory was specified")
1599+
end
1600+
1601+
data("include additional YAML configuration with relative conf.d" => [{"relative_path" => true}],
1602+
"include additional YAML configuration with full path conf.d" => [{"relative_path" => false}])
1603+
test "additional relative configuration file (conf.d/child.yml) was loaded" do |(option)|
1604+
conf_dir = option["relative_path"] ? "conf.d" : "#{@tmp_dir}/conf.d"
1605+
conf_path = create_config_include_dir_configuration("parent.yml", conf_dir, true)
1606+
create_conf_file('conf.d/child.yml', "")
1607+
assert_log_matches(create_cmdline(conf_path),
1608+
"[info]: loading additional configuration file path=\"#{conf_dir}/child.yml\"")
1609+
end
1610+
end
15391611
end

‎test/test_supervisor.rb‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1054,6 +1054,129 @@ def test_stop_parallel_old_supervisor_after_delay
10541054
end
10551055
end
10561056

1057+
sub_test_case "include additional configuration" do
1058+
setup do
1059+
@config_include_dir = File.join(@tmp_dir, "conf.d")
1060+
FileUtils.mkdir_p(@config_include_dir)
1061+
end
1062+
1063+
test "no additional configuration" do
1064+
c = Fluent::Config::Element.new('system', '', { 'config_include_dir' => '' }, [])
1065+
stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) }
1066+
supervisor = Fluent::Supervisor.new({})
1067+
stub(supervisor).build_spawn_command { "dummy command line" }
1068+
supervisor.configure(supervisor: true)
1069+
assert_equal([c], supervisor.instance_variable_get(:@conf).elements)
1070+
end
1071+
1072+
data(
1073+
"single source" => ["forward"],
1074+
"multiple sources" => ["forward", "tcp"])
1075+
test "additional configuration" do |sources|
1076+
c = Fluent::Config::Element.new('system', '',
1077+
{ 'config_include_dir' => @config_include_dir }, [])
1078+
config_path = "#{@config_include_dir}/dummy.conf"
1079+
stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8",
1080+
additional_config: anything, use_v1_config: anything,
1081+
type: anything) { config_element('ROOT', '', {}, [c]) }
1082+
sources.each do |type|
1083+
config = <<~EOF
1084+
<source>
1085+
@type #{type}
1086+
</source>
1087+
EOF
1088+
additional_config_path = "#{@config_include_dir}/#{type}.yml"
1089+
write_config(additional_config_path, config)
1090+
stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8",
1091+
use_v1_config: true, type: :guess) {
1092+
Fluent::Config.parse(File.read(additional_config_path), File.dirname(config_path), true)
1093+
}
1094+
end
1095+
supervisor = Fluent::Supervisor.new({})
1096+
stub(supervisor).build_spawn_command { "dummy command line" }
1097+
supervisor.configure(supervisor: true)
1098+
expected = [c].concat(sources.collect { |type| {"@type" => type} })
1099+
assert_equal(expected, supervisor.instance_variable_get(:@conf).elements)
1100+
end
1101+
1102+
data(
1103+
"single YAML source" => ["forward"],
1104+
"multiple YAML sources" => ["forward", "tcp"])
1105+
test "additional YAML configuration" do |sources|
1106+
c = Fluent::Config::Element.new('system', '',
1107+
{ 'config_include_dir' => @config_include_dir }, [])
1108+
config_path = "#{@config_include_dir}/dummy.yml"
1109+
stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8",
1110+
additional_config: anything, use_v1_config: anything,
1111+
type: anything) { config_element('ROOT', '', {}, [c]) }
1112+
sources.each do |type|
1113+
config = <<~EOF
1114+
config:
1115+
- source:
1116+
$type: #{type}
1117+
EOF
1118+
additional_config_path = "#{@config_include_dir}/#{type}.yml"
1119+
write_config(additional_config_path, config)
1120+
stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8",
1121+
use_v1_config: true, type: :guess) {
1122+
Fluent::Config::YamlParser.parse(additional_config_path)
1123+
}
1124+
end
1125+
supervisor = Fluent::Supervisor.new({})
1126+
stub(supervisor).build_spawn_command { "dummy command line" }
1127+
supervisor.configure(supervisor: true)
1128+
expected = [c].concat(sources.collect { |type| {"@type" => type} })
1129+
assert_equal(expected, supervisor.instance_variable_get(:@conf).elements)
1130+
end
1131+
1132+
data(
1133+
"single source" => [false, ["forward"]],
1134+
"multiple sources" => [false, ["forward", "tcp"]],
1135+
"single YAML source" => [true, ["forward"]],
1136+
"multiple YAML sources" => [true, ["forward", "tcp"]])
1137+
test "reload with additional configuration" do |(yaml, sources)|
1138+
c = Fluent::Config::Element.new('system', '',
1139+
{ 'config_include_dir' => @config_include_dir }, [])
1140+
config_path = "#{@config_include_dir}/dummy.yml"
1141+
stub(Fluent::Config).build(config_path: "/etc/fluent/fluent.conf", encoding: "utf-8",
1142+
additional_config: anything, use_v1_config: anything,
1143+
type: anything) { config_element('ROOT', '', {}, [c]) }
1144+
sources.each do |type|
1145+
if yaml
1146+
config = <<~EOF
1147+
config:
1148+
- source:
1149+
$type: #{type}
1150+
EOF
1151+
additional_config_path = "#{@config_include_dir}/#{type}.yml"
1152+
write_config(additional_config_path, config)
1153+
stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8",
1154+
use_v1_config: true, type: :guess) {
1155+
Fluent::Config::YamlParser.parse(additional_config_path)
1156+
}
1157+
else
1158+
config = <<~EOF
1159+
<source>
1160+
@type #{type}
1161+
</source>
1162+
EOF
1163+
additional_config_path = "#{@config_include_dir}/#{type}.conf"
1164+
write_config(additional_config_path, config)
1165+
stub(Fluent::Config).build(config_path: additional_config_path, encoding: "utf-8",
1166+
use_v1_config: true, type: :guess) {
1167+
Fluent::Config.parse(File.read(additional_config_path), File.dirname(config_path), true)
1168+
}
1169+
end
1170+
end
1171+
supervisor = Fluent::Supervisor.new({})
1172+
stub(supervisor).build_spawn_command { "dummy command line" }
1173+
supervisor.configure(supervisor: true)
1174+
supervisor.__send__(:reload_config)
1175+
expected = [c].concat(sources.collect { |type| {"@type" => type} })
1176+
assert_equal(expected, supervisor.instance_variable_get(:@conf).elements)
1177+
end
1178+
end
1179+
10571180
def create_debug_dummy_logger
10581181
dl_opts = {}
10591182
dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG

0 commit comments

Comments
 (0)