From 8a26010d4f20e248bc16bbb04714522638b623bf Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Sun, 18 Mar 2018 16:52:21 +0000 Subject: [PATCH 01/16] Test against latest td-agent versions --- Rakefile | 7 +------ test/docker/Dockerfile.tdagent-centos | 6 +++--- test/docker/Dockerfile.tdagent-ubuntu | 4 ++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Rakefile b/Rakefile index 9b6a0a2..433f63b 100644 --- a/Rakefile +++ b/Rakefile @@ -22,12 +22,7 @@ namespace :docker do distros.each do |distro| task distro do puts "testing on #{distro}" - begin - FileUtils.cp("test/docker/Dockerfile.#{distro}", 'Dockerfile') - sh 'docker build .' - ensure - FileUtils.rm('Dockerfile') - end + sh "docker build . -f test/docker/Dockerfile.#{distro}" end end end diff --git a/test/docker/Dockerfile.tdagent-centos b/test/docker/Dockerfile.tdagent-centos index bab1d4c..8810304 100644 --- a/test/docker/Dockerfile.tdagent-centos +++ b/test/docker/Dockerfile.tdagent-centos @@ -1,8 +1,8 @@ -FROM centos:7.1.1503 +FROM centos:7 RUN yum remove -y fakesystemd \ && rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent \ - && printf "[treasuredata]\nname=TreasureData\nbaseurl=http://packages.treasuredata.com/2/redhat/\$releasever/\$basearch\ngpgcheck=1\ngpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent\n" > /etc/yum.repos.d/td.repo \ + && printf "[treasuredata]\nname=TreasureData\nbaseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch\ngpgcheck=1\ngpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent\n" > /etc/yum.repos.d/td.repo \ && yum install -y td-agent make gcc-c++ systemd ENV PATH /opt/td-agent/embedded/bin/:$PATH @@ -12,4 +12,4 @@ COPY Gemfile ./ COPY fluent-plugin-systemd.gemspec ./ RUN bundle install COPY . . -RUN rake test TESTOPTS="-v" +RUN bundle exec rake test TESTOPTS="-v" diff --git a/test/docker/Dockerfile.tdagent-ubuntu b/test/docker/Dockerfile.tdagent-ubuntu index f03144a..47e016c 100644 --- a/test/docker/Dockerfile.tdagent-ubuntu +++ b/test/docker/Dockerfile.tdagent-ubuntu @@ -7,7 +7,7 @@ RUN apt-get update -q \ ca-certificates \ libsystemd0 \ && curl https://packages.treasuredata.com/GPG-KEY-td-agent | apt-key add - \ - && echo "deb http://packages.treasuredata.com/2/ubuntu/xenial/ xenial contrib" > /etc/apt/sources.list.d/treasure-data.list \ + && echo "deb http://packages.treasuredata.com/3/ubuntu/xenial/ xenial contrib" > /etc/apt/sources.list.d/treasure-data.list \ && apt-get update \ && apt-get install -y td-agent \ && apt-get clean \ @@ -22,4 +22,4 @@ COPY Gemfile ./ COPY fluent-plugin-systemd.gemspec ./ RUN bundle check || bundle install COPY . . -RUN rake test TESTOPTS="-v" +RUN bundle exec rake test TESTOPTS="-v" From 88c64bd89f030fcd7a030ab2885034c9f47057e4 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Mon, 19 Mar 2018 11:15:25 +0000 Subject: [PATCH 02/16] Remove legacy PosWriter The PosWriter class was needed when we needed to handle pos files manualy. And then to migrate to a storage. We can now remove this additional class. --- lib/fluent/plugin/in_systemd.rb | 14 +-- lib/fluent/plugin/systemd/pos_writer.rb | 99 --------------- test/plugin/systemd/test_pos_writer.rb | 157 ------------------------ test/plugin/test_in_systemd.rb | 57 ++++----- 4 files changed, 27 insertions(+), 300 deletions(-) delete mode 100644 lib/fluent/plugin/systemd/pos_writer.rb delete mode 100644 test/plugin/systemd/test_pos_writer.rb diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb index 000f153..b926446 100644 --- a/lib/fluent/plugin/in_systemd.rb +++ b/lib/fluent/plugin/in_systemd.rb @@ -2,13 +2,12 @@ require 'systemd/journal' require 'fluent/plugin/input' -require 'fluent/plugin/systemd/pos_writer' require 'fluent/plugin/systemd/entry_mutator' module Fluent module Plugin # Fluentd plugin for reading from the systemd journal - class SystemdInput < Input # rubocop:disable Metrics/ClassLength + class SystemdInput < Input Fluent::Plugin.register_input('systemd', self) helpers :timer, :storage @@ -17,7 +16,6 @@ class SystemdInput < Input # rubocop:disable Metrics/ClassLength config_param :path, :string, default: '/var/log/journal' config_param :filters, :array, default: [] - config_param :pos_file, :string, default: nil, deprecated: "Use section with `persistent: true' instead" config_param :read_from_head, :bool, default: false config_param :strip_underscores, :bool, default: false, deprecated: 'Use section or `systemd_entry` ' \ 'filter plugin instead' @@ -39,7 +37,7 @@ class SystemdInput < Input # rubocop:disable Metrics/ClassLength def configure(conf) super @journal = nil - @pos_storage = PosWriter.new(@pos_file, storage_create(usage: 'positions')) + @pos_storage = storage_create(usage: 'positions') # legacy strip_underscores backwards compatibility (legacy takes # precedence and is mutually exclusive with the entry block) mut_opts = @strip_underscores ? { fields_strip_underscores: true } : @entry_opts.to_h @@ -49,15 +47,9 @@ def configure(conf) def start super - @pos_storage.start timer_execute(:in_systemd_emit_worker, 1, &method(:run)) end - def shutdown - @pos_storage.shutdown - super - end - private def init_journal @@ -80,7 +72,7 @@ def seek seek_to(cursor || read_from) rescue Systemd::JournalError log.warn( - "Could not seek to cursor #{cursor} found in pos file: #{@pos_storage.path}, " \ + "Could not seek to cursor #{cursor} found in position file: #{@pos_storage.path}, " \ "falling back to reading from #{read_from}" ) seek_to(read_from) diff --git a/lib/fluent/plugin/systemd/pos_writer.rb b/lib/fluent/plugin/systemd/pos_writer.rb deleted file mode 100644 index db09027..0000000 --- a/lib/fluent/plugin/systemd/pos_writer.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true - -require 'fluent/plugin/input' - -module Fluent - module Plugin - class SystemdInput < Input - # This is used to write the systemd cursor to the configured storage - # We do this periodicly in a thread so as to not contend on resources - # that might be needed for more important tasks. - # - # When signaled to shutdown we ensure that the most recent cursor - # has been written. - # - # If fluentd stops runnning without cleanly shutting down PosWriter - # the cursor could be up to 1 second stale - class PosWriter - def initialize(pos_file, storage) - @path = pos_file - @lock = Mutex.new - @storage = storage - @cursor = nil - @written_cursor = nil - setup - end - - def get(key) - @storage ? @storage.get(key) : @cursor - end - - def put(key, cursor) - return @storage.put(key, cursor) if @storage - @lock.synchronize { @cursor = cursor } - end - - def path - @path || @storage.path - end - - def start - return unless @path - @running = true - @thread = Thread.new(&method(:work)) - end - - def shutdown - return unless @path - @running = false - @thread.join - write_pos - end - - private - - def setup - if @storage.persistent - migrate_to_storage - elsif @path - @cursor = read_legacy_pos if legacy_file? - @storage = nil - end - end - - def legacy_file? - @path && File.exist?(@path) - end - - def read_legacy_pos - IO.read(@path).chomp - end - - def migrate_to_storage - return unless legacy_file? - @storage.put(:journal, read_legacy_pos) - File.delete(@path) - @path = nil - end - - def work - while @running - write_pos - sleep 1 - end - end - - def write_pos - @lock.synchronize do - if @written_cursor != @cursor - file = File.open(@path, 'w+', 0o644) - file.print @cursor - file.close - @written_cursor = @cursor - end - end - end - end - end - end -end diff --git a/test/plugin/systemd/test_pos_writer.rb b/test/plugin/systemd/test_pos_writer.rb deleted file mode 100644 index 338e0ff..0000000 --- a/test/plugin/systemd/test_pos_writer.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: true - -require_relative '../../helper' -require 'tempfile' -require 'fluent/plugin/systemd/pos_writer' - -class SystemdInputTest < Test::Unit::TestCase - class FakeStorage - def initialize(options) - @persistent = options[:persistent] - @store = {} - end - - attr_reader :persistent - - def put(key, value) - @store[key] = value - end - - def get(key) - @store[key] - end - end - - def storage(options = {}) - FakeStorage.new(options) - end - - def test_reading_the_cursor_when_file_exists - pos_file = Tempfile.new('foo.pos') - pos_file.write('cursor_value') - pos_file.close - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(pos_file.path, storage) - assert_equal pos_writer.get(:journal), 'cursor_value' - pos_file.unlink - end - - def test_reading_the_cursor_when_file_does_not_exist_yet - dir = Dir.mktmpdir('posdir') - path = "#{dir}/foo.pos" - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage) - assert_equal pos_writer.get(:journal), nil - FileUtils.rm_rf dir - end - - def test_reading_the_cusor_when_the_path_is_nil - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(nil, storage) - assert_equal pos_writer.get(:journal), nil - pos_writer.put(:journal, 'a_cursor') - assert_equal pos_writer.get(:journal), 'a_cursor' - end - - def test_writing_the_cursor_when_file_does_not_exist_yet - dir = Dir.mktmpdir('posdir') - path = "#{dir}/foo.pos" - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage) - pos_writer.start - pos_writer.put(:journal, 'this is the cursor') - assert_equal pos_writer.get(:journal), 'this is the cursor' - sleep 1 - assert_equal File.read(path), 'this is the cursor' - FileUtils.rm_rf dir - end - - def test_file_permission_when_file_does_not_exist_yet - dir = Dir.mktmpdir('posdir') - path = "#{dir}/foo.pos" - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage) - pos_writer.start - pos_writer.put(:journal, 'this is the cursor') - sleep 1 - assert_equal format('%o', File::Stat.new(path).mode)[-4, 4], '0644' # rubocop:disable Style/FormatStringToken - FileUtils.rm_rf dir - end - - def test_writing_the_cursor_when_the_writer_is_shutdown - dir = Dir.mktmpdir('posdir') - path = "#{dir}/foo.pos" - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage) - pos_writer.start - pos_writer.put(:journal, 'this is the cursor') - pos_writer.shutdown - assert_equal File.read(path), 'this is the cursor' - FileUtils.rm_rf dir - end - - def test_writing_the_cursor_when_the_file_exists - pos_file = Tempfile.new('foo.pos') - pos_file.write('cursor_value') - pos_file.close - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(pos_file.path, storage) - assert_equal pos_writer.get(:journal), 'cursor_value' - pos_writer.start - pos_writer.put(:journal, 'this is the cursor') - sleep 1 - assert_equal File.read(pos_file.path), 'this is the cursor' - pos_file.unlink - end - - def test_writing_and_then_reading_the_pos_roundtrip - dir = Dir.mktmpdir('posdir') - path = "#{dir}/foo.pos" - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage) - pos_writer.start - pos_writer.put(:journal, 'this is the cursor') - pos_writer.shutdown - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage) - assert_equal pos_writer.get(:journal), 'this is the cursor' - FileUtils.rm_rf dir - end - - def test_upgrading_from_pos_writer_to_storage - store = storage(persistent: true) - pos_file = Tempfile.new('foo.pos') - pos_file.write('cursor_value') - pos_file.close - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(pos_file.path, store) - - # It removes the old file - assert !File.exist?(pos_file.path) - - # it copies the value to the store - assert_equal store.get(:journal), 'cursor_value' - - # it uses the store - pos_writer.put(:journal, 'new_value') - assert_equal store.get(:journal), 'new_value' - - # start and shutdown should be noops - pos_writer.put(:journal, 'another_value') - assert_nil pos_writer.start - assert_nil pos_writer.shutdown - assert_equal pos_writer.get(:journal), 'another_value' - assert_equal store.get(:journal), 'another_value' - end - - def test_when_the_old_pos_writer_file_does_not_exist - store = storage(persistent: true) - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new('not_a_real_path_to_a_file', store) - assert_nil store.get(:journal) - - # it works - pos_writer.put(:journal, 'new_value') - assert_equal store.get(:journal), 'new_value' - assert_equal pos_writer.get(:journal), 'new_value' - end - - def test_when_no_pos_file_path_given - # uses storage even if not persistent - store = storage(persistent: false) - pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(nil, store) - - pos_writer.put(:journal, 'new_value') - assert_equal store.get(:journal), 'new_value' - assert_equal pos_writer.get(:journal), 'new_value' - end -end diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb index dee4747..281e5b6 100644 --- a/test/plugin/test_in_systemd.rb +++ b/test/plugin/test_in_systemd.rb @@ -70,17 +70,17 @@ def setup strip_underscores true ) - pos_dir = Dir.mktmpdir('posdir') - - @pos_path = "#{pos_dir}/foo.pos" - - @pos_config = base_config + %( - pos_file #{@pos_path} + @storage_path = File.join(Dir.mktmpdir('pos_dir'), 'storage.json') + + @storage_config = @base_config + %( + + @type local + persistent true + path #{storage_path} + ) - @storage_path = File.join(pos_dir.to_s, 'storage.json') - - @head_config = @pos_config + %( + @head_config = @storage_config + %( read_from_head true ) @@ -88,7 +88,7 @@ def setup filters [{ "_SYSTEMD_UNIT": "systemd-journald.service" }] ) - @tail_config = @pos_config + %( + @tail_config = @storage_config + %( read_from_head false ) @@ -98,14 +98,22 @@ def setup ) end - attr_reader :journal, :base_config, :pos_path, :pos_config, :head_config, + attr_reader :journal, :base_config, :head_config, :filter_config, :strip_config, :tail_config, :not_present_config, - :badmsg_config, :storage_path + :badmsg_config, :storage_path, :storage_config def create_driver(config) Fluent::Test::Driver::Input.new(Fluent::Plugin::SystemdInput).configure(config) end + def read_pos + JSON.parse(File.read(storage_path))['journal'] + end + + def write_pos(pos) + File.write(storage_path, JSON.dump(journal: pos)) + end + def test_configure_requires_tag assert_raise Fluent::ConfigError do create_driver('') @@ -154,22 +162,9 @@ def test_reading_from_the_journal_tail_with_strip_underscores_legacy end def test_storage_file_is_written - storage_config = config_element('ROOT', '', { - 'tag' => 'test', - 'path' => 'test/fixture', - '@id' => 'test-01' - }, [ - config_element('storage', '', - '@type' => 'local', - 'persistent' => true, - 'path' => @storage_path) - ]) - d = create_driver(storage_config) d.run(expect_emits: 1) - storage = JSON.parse(File.read(storage_path)) - result = storage['journal'] - assert_equal result, 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=1cd;b=4737ffc504774b3ba67020bc947f1bc0;m=42f2dd;t=4d905e4cd5a92;x=25b3f86ff2774ac4' + assert_equal 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=1cd;b=4737ffc504774b3ba67020bc947f1bc0;m=42f2dd;t=4d905e4cd5a92;x=25b3f86ff2774ac4', read_pos end def test_reading_from_head @@ -212,9 +207,7 @@ def test_reading_with_filters end def test_reading_from_a_pos - file = File.open(pos_path, 'w+') - file.print 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=13f;b=4737ffc504774b3ba67020bc947f1bc0;m=ffadd;t=4d905e49a6291;x=9a11dd9ffee96e9f' - file.close + write_pos 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=13f;b=4737ffc504774b3ba67020bc947f1bc0;m=ffadd;t=4d905e49a6291;x=9a11dd9ffee96e9f' d = create_driver(head_config) d.end_if do d.events.size >= 142 @@ -224,9 +217,7 @@ def test_reading_from_a_pos end def test_reading_from_an_invalid_pos - file = File.open(pos_path, 'w+') - file.print 'thisisinvalid' - file.close + write_pos 'thisisinvalid' # It continues as if the pos file did not exist d = create_driver(head_config) @@ -236,7 +227,7 @@ def test_reading_from_an_invalid_pos d.run(timeout: 5) assert_equal 461, d.events.size assert_match( - "Could not seek to cursor thisisinvalid found in pos file: #{pos_path}, falling back to reading from head", + "Could not seek to cursor thisisinvalid found in position file: #{storage_path}, falling back to reading from head", d.logs.last ) end From e39113242b96d8842114e4e0c12cdcc91e99c26e Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Mon, 19 Mar 2018 11:22:06 +0000 Subject: [PATCH 03/16] Remove deprecated strip_underscores config option --- lib/fluent/plugin/in_systemd.rb | 7 +------ test/plugin/test_in_systemd.rb | 19 +------------------ 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb index b926446..9c83980 100644 --- a/lib/fluent/plugin/in_systemd.rb +++ b/lib/fluent/plugin/in_systemd.rb @@ -17,8 +17,6 @@ class SystemdInput < Input config_param :path, :string, default: '/var/log/journal' config_param :filters, :array, default: [] config_param :read_from_head, :bool, default: false - config_param :strip_underscores, :bool, default: false, deprecated: 'Use section or `systemd_entry` ' \ - 'filter plugin instead' config_param :tag, :string config_section :storage do @@ -38,10 +36,7 @@ def configure(conf) super @journal = nil @pos_storage = storage_create(usage: 'positions') - # legacy strip_underscores backwards compatibility (legacy takes - # precedence and is mutually exclusive with the entry block) - mut_opts = @strip_underscores ? { fields_strip_underscores: true } : @entry_opts.to_h - @mutator = SystemdEntryMutator.new(**mut_opts) + @mutator = SystemdEntryMutator.new(**@entry_opts.to_h) @mutator.warnings.each { |warning| log.warn(warning) } end diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb index 281e5b6..8b865dd 100644 --- a/test/plugin/test_in_systemd.rb +++ b/test/plugin/test_in_systemd.rb @@ -65,11 +65,6 @@ def setup read_from_head true ) - # deprecated - @strip_config = base_config + %( - strip_underscores true - ) - @storage_path = File.join(Dir.mktmpdir('pos_dir'), 'storage.json') @storage_config = @base_config + %( @@ -99,7 +94,7 @@ def setup end attr_reader :journal, :base_config, :head_config, - :filter_config, :strip_config, :tail_config, :not_present_config, + :filter_config, :tail_config, :not_present_config, :badmsg_config, :storage_path, :storage_config def create_driver(config) @@ -149,18 +144,6 @@ def test_reading_from_the_journal_tail_mutate_entry(data) assert_equal(expected, d.events) end - # deprecated config option for backwards compatibility - def test_reading_from_the_journal_tail_with_strip_underscores_legacy - d = create_driver(strip_config) - expected = [[ - 'test', - 1_364_519_243, - EntryTestData::EXPECTED[:fields_strip_underscores] - ]] - d.run(expect_emits: 1) - assert_equal(expected, d.events) - end - def test_storage_file_is_written d = create_driver(storage_config) d.run(expect_emits: 1) From 78bfdb2393fa955f9e6a2a8bee44a46e37f9b3ba Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Mon, 19 Mar 2018 11:30:41 +0000 Subject: [PATCH 04/16] First prerelease --- fluent-plugin-systemd.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-systemd.gemspec b/fluent-plugin-systemd.gemspec index c347870..611a29c 100644 --- a/fluent-plugin-systemd.gemspec +++ b/fluent-plugin-systemd.gemspec @@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = 'fluent-plugin-systemd' - spec.version = '0.3.1' + spec.version = '1.0.0.rc1' spec.authors = ['Ed Robinson'] spec.email = ['edward-robinson@cookpad.com'] From d681e78e66f33c2f2767bdcc02a24a5cd3313765 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Sat, 14 Apr 2018 13:15:14 +0100 Subject: [PATCH 05/16] Rename filters to matches I felt the term filters was being somewhat overloaded since fluentd also has a concept of filters. In the journalctl documentation these expressions are refered to as matches... so it seems resonable to do the same... The filters param will continue to work, with a warning... --- lib/fluent/plugin/in_systemd.rb | 5 +++-- test/plugin/test_in_systemd.rb | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb index 9c83980..f58403d 100644 --- a/lib/fluent/plugin/in_systemd.rb +++ b/lib/fluent/plugin/in_systemd.rb @@ -15,7 +15,8 @@ class SystemdInput < Input DEFAULT_STORAGE_TYPE = 'local' config_param :path, :string, default: '/var/log/journal' - config_param :filters, :array, default: [] + config_param :filters, :array, default: [], deprecated: 'filters has been renamed as matches' + config_param :matches, :array, default: nil config_param :read_from_head, :bool, default: false config_param :tag, :string @@ -54,7 +55,7 @@ def init_journal # make sure initial call to wait doesn't return :invalidate # see https://github.com/ledbettj/systemd-journal/issues/70 @journal.wait(0) - @journal.filter(*@filters) + @journal.filter(*(@matches || @filters)) seek true rescue Systemd::JournalError => e diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb index 8b865dd..e5745f5 100644 --- a/test/plugin/test_in_systemd.rb +++ b/test/plugin/test_in_systemd.rb @@ -83,6 +83,10 @@ def setup filters [{ "_SYSTEMD_UNIT": "systemd-journald.service" }] ) + @matches_config = @head_config + %( + matches [{ "_SYSTEMD_UNIT": "systemd-journald.service" }] + ) + @tail_config = @storage_config + %( read_from_head false ) @@ -94,7 +98,7 @@ def setup end attr_reader :journal, :base_config, :head_config, - :filter_config, :tail_config, :not_present_config, + :matches_config, :filter_config, :tail_config, :not_present_config, :badmsg_config, :storage_path, :storage_config def create_driver(config) @@ -180,6 +184,7 @@ def test_backoff_on_buffer_error d.run(expect_emits: 1) end + # deprecated and replaced with matches def test_reading_with_filters d = create_driver(filter_config) d.end_if do @@ -189,6 +194,15 @@ def test_reading_with_filters assert_equal 3, d.events.size end + def test_reading_with_matches + d = create_driver(matches_config) + d.end_if do + d.events.size >= 3 + end + d.run(timeout: 5) + assert_equal 3, d.events.size + end + def test_reading_from_a_pos write_pos 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=13f;b=4737ffc504774b3ba67020bc947f1bc0;m=ffadd;t=4d905e49a6291;x=9a11dd9ffee96e9f' d = create_driver(head_config) From 75e7ae9d48b3231054a30e9039ac218e8969b546 Mon Sep 17 00:00:00 2001 From: Joel Gerber Date: Fri, 13 Apr 2018 11:22:47 -0400 Subject: [PATCH 06/16] Document deprecation/replacement of filters parameter with matches I believe this commit should properly state that the filters parameter is getting deprecated/replaced with the matches parameter. This includes renaming the Filtering-Details.md document to Matching-Details.md, and changing the wording to be accurate given the new parameter name. --- README.md | 14 +++--- ...ltering-Details.md => Matching-Details.md} | 45 ++++++++++--------- 2 files changed, 33 insertions(+), 26 deletions(-) rename docs/{Filtering-Details.md => Matching-Details.md} (75%) diff --git a/README.md b/README.md index 73a5698..9f41322 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ or @type systemd tag kube-proxy path /var/log/journal - filters [{ "_SYSTEMD_UNIT": "kube-proxy.service" }] + matches [{ "_SYSTEMD_UNIT": "kube-proxy.service" }] read_from_head true @type local @@ -65,11 +65,15 @@ Path to the systemd journal, defaults to `/var/log/journal` **`filters`** -Expects an array of hashes defining desired filters to apply to all log -messages. When this property is not specified, this plugin will default to -having no filters specified. +_This parameter name is deprecated and should be renamed to `matches`_ -See [filtering details](docs/Filtering-Details.md) for a more exhaustive +**`matches`** + +Expects an array of hashes defining desired matches to filter the log +messages with. When this property is not specified, this plugin will default to +reading all logs from the journal. + +See [matching details](docs/Matching-Details.md) for a more exhaustive description of this property and how to use it. **`pos_file`** diff --git a/docs/Filtering-Details.md b/docs/Matching-Details.md similarity index 75% rename from docs/Filtering-Details.md rename to docs/Matching-Details.md index a2a12ec..ac72e29 100644 --- a/docs/Filtering-Details.md +++ b/docs/Matching-Details.md @@ -1,35 +1,38 @@ -# Filtering Details +# Matching Details ## Overview -This application takes an array of hashes passed to the `filters` parameter +This application takes an array of hashes passed to the `matches` parameter within a `systemd` typed source definition in your `fluent.conf` configuration file and then parses them into a format understood by `libsystemd`'s journal API. The basis behind what `libsystemd`'s API expects can be found documented in the `journalctl` [man page](https://www.freedesktop.org/software/systemd/man/journalctl.html). +The result of this is that only logs which match the defined set of matching +rules will be further processed. + ## Usage Information -In order to utilize this plugin's filtering capabilities, you will need to +In order to utilize this plugin's matching capabilities, you will need to understand how this plugin transforms the passed array of hashes into a format that is understood by `libsystemd`. The best way to describe this process is probably by example. The following sub-sections lists out various scenarios that you might wish to perform with -this plugin's filtering mechanism and describes both how to configure them, +this plugin's matching mechanism and describes both how to configure them, while also mapping them to examples from the `journalctl` [man page](https://www.freedesktop.org/software/systemd/man/journalctl.html). ### No Filters -You can leave the `filters` property out altogether, or include a `filters` -property with an empty array (as shown below) to specify that no filtering +You can leave the `matches` property out altogether, or include a `matches` +property with an empty array (as shown below) to specify that no matching should occur. - filters [] + matches [] -Which matches this part of the `journalctl` man page: +Which coincides with this part of the `journalctl` man page: > Without arguments, all collected logs are shown unfiltered: > @@ -37,13 +40,13 @@ Which matches this part of the `journalctl` man page: ### Single Filter -You can pass a single hash map to the `filters` array with a single key/value -pair specified to filter out all log entries that do not match the given -field/value combination. +You can pass a single hash map to the `matches` array with a single key/value +pair specified to only process log entries that match the given field/value +combination. For example: - filters [{"_SYSTEMD_UNIT": "avahi-daemon.service"}] + matches [{"_SYSTEMD_UNIT": "avahi-daemon.service"}] Which coincides with this part of the the `journalctl` man page: @@ -54,9 +57,9 @@ Which coincides with this part of the the `journalctl` man page: ### Multi-Field Filters -You can pass a single hash map to the `filters` array with multiple key/value -pairs to filter out all log entries that do not match the combination of all of -the specified key/value combinations. +You can pass a single hash map to the `matches` array with multiple key/value +pairs to only process log entries that match the combination of all of the +specified key/value combinations. The passed key/value pairs are treated as a logical `AND`, such that all of the pairs must be true in order to allow further processing of the current log @@ -64,7 +67,7 @@ entry. For Example: - filters [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}] + matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}] Which coincides with this part of the the `journalctl` man page: @@ -74,15 +77,15 @@ Which coincides with this part of the the `journalctl` man page: > `journalctl _SYSTEMD_UNIT=avahi-daemon.service _PID=28097` You can also perform a logical `OR` by splitting key/value pairs across multiple -hashes passed to the `filters` array like so: +hashes passed to the `matches` array like so: - filters [{"_SYSTEMD_UNIT": "avahi-daemon.service"}, {"_PID": 28097}] + matches [{"_SYSTEMD_UNIT": "avahi-daemon.service"}, {"_PID": 28097}] You can combine both `AND` and `OR` combinations together; using a single hash map to define conditions that `AND` together and using multiple hash maps to define conditions that `OR` together like so: - filters [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}] + matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}] This can be expressed in psuedo-code like so: @@ -105,7 +108,7 @@ Fields with arrays as values are treated as a logical `OR` statement. For example: - filters [{"_SYSTEMD_UNIT": ["avahi-daemon.service", "dbus.service"]}] + matches [{"_SYSTEMD_UNIT": ["avahi-daemon.service", "dbus.service"]}] Which coincides with this part of the `journalctl` man page: @@ -119,7 +122,7 @@ particularly helpful when you want to create aggregate logic For example: - filters [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}] + matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}] This can be expressed in psuedo-code like so: From b7a08166a793b5c07d473345e678f8b2c739ce03 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Sat, 14 Apr 2018 13:40:08 +0100 Subject: [PATCH 07/16] Second prerelease --- fluent-plugin-systemd.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-systemd.gemspec b/fluent-plugin-systemd.gemspec index 611a29c..9718b3c 100644 --- a/fluent-plugin-systemd.gemspec +++ b/fluent-plugin-systemd.gemspec @@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = 'fluent-plugin-systemd' - spec.version = '1.0.0.rc1' + spec.version = '1.0.0.rc2' spec.authors = ['Ed Robinson'] spec.email = ['edward-robinson@cookpad.com'] From b6232b3f95c7ffefb6ed088476e41039c9621416 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 3 May 2018 20:19:28 +0100 Subject: [PATCH 08/16] Revert the code introduced by #41 --- lib/fluent/plugin/in_systemd.rb | 7 +------ test/plugin/test_in_systemd.rb | 15 +-------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb index f58403d..e5ef553 100644 --- a/lib/fluent/plugin/in_systemd.rb +++ b/lib/fluent/plugin/in_systemd.rb @@ -118,12 +118,7 @@ def formatted(entry) def watch while @journal.move_next - begin - yield @journal.current_entry - rescue Systemd::JournalError => e - log.warn("Error Parsing Journal: #{e.class}: #{e.message}") - next - end + yield @journal.current_entry @pos_storage.put(:journal, @journal.cursor) end end diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb index e5745f5..d411991 100644 --- a/test/plugin/test_in_systemd.rb +++ b/test/plugin/test_in_systemd.rb @@ -59,12 +59,6 @@ def setup path test/fixture ) - @badmsg_config = %( - tag test - path test/fixture/corrupt - read_from_head true - ) - @storage_path = File.join(Dir.mktmpdir('pos_dir'), 'storage.json') @storage_config = @base_config + %( @@ -99,7 +93,7 @@ def setup attr_reader :journal, :base_config, :head_config, :matches_config, :filter_config, :tail_config, :not_present_config, - :badmsg_config, :storage_path, :storage_config + :storage_path, :storage_config def create_driver(config) Fluent::Test::Driver::Input.new(Fluent::Plugin::SystemdInput).configure(config) @@ -246,11 +240,4 @@ def test_journal_not_present d.run(timeout: 5) assert_match 'Systemd::JournalError: No such file or directory retrying in 1s', d.logs.last end - - def test_continue_on_bad_message - d = create_driver(badmsg_config) - d.run(expect_emits: 460) - assert_equal 460, d.events.size - assert_equal 0, d.error_events.size - end end From dee3e7a8d68f4b3afb909ceec560444e75e7ca69 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 3 May 2018 20:37:36 +0100 Subject: [PATCH 09/16] Skip corrupt systemd entries This is a reworked solution to #16 Previously addressed by #41... which was removed for licensing reasons see #60 for background. --- lib/fluent/plugin/in_systemd.rb | 14 +++++++++----- test/plugin/test_in_systemd.rb | 14 +++++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb index e5ef553..43e69f4 100644 --- a/lib/fluent/plugin/in_systemd.rb +++ b/lib/fluent/plugin/in_systemd.rb @@ -116,11 +116,15 @@ def formatted(entry) @mutator.run(entry) end - def watch - while @journal.move_next - yield @journal.current_entry - @pos_storage.put(:journal, @journal.cursor) - end + def watch(&block) + yield_current_entry(&block) while @journal.move_next + end + + def yield_current_entry + yield @journal.current_entry + @pos_storage.put(:journal, @journal.cursor) + rescue Systemd::JournalError => e + log.warn("Error reading from Journal: #{e.class}: #{e.message}") end end end diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb index d411991..b00a517 100644 --- a/test/plugin/test_in_systemd.rb +++ b/test/plugin/test_in_systemd.rb @@ -89,11 +89,17 @@ def setup tag test path test/not_a_real_path ) + + @corrupt_entries_config = %( + tag test + path test/fixture/corrupt + read_from_head true + ) end attr_reader :journal, :base_config, :head_config, :matches_config, :filter_config, :tail_config, :not_present_config, - :storage_path, :storage_config + :storage_path, :storage_config, :corrupt_entries_config def create_driver(config) Fluent::Test::Driver::Input.new(Fluent::Plugin::SystemdInput).configure(config) @@ -240,4 +246,10 @@ def test_journal_not_present d.run(timeout: 5) assert_match 'Systemd::JournalError: No such file or directory retrying in 1s', d.logs.last end + + def test_reading_from_a_journal_with_corrupted_entries + d = create_driver(corrupt_entries_config) + d.run(expect_emits: 460) + assert_equal 460, d.events.size + end end From b9c0425c8b9351a4e0268fa1a91d132cd3a615cc Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 3 May 2018 20:56:33 +0100 Subject: [PATCH 10/16] Relicense as Apache 2.0 See the discussion in #60 for aproval of this by all contibutors who's code is currently included in the plugin. --- LICENCE | 209 +++++++++++++++++++-- README.md | 2 +- fluent-plugin-systemd.gemspec | 2 +- lib/fluent/plugin/filter_systemd_entry.rb | 14 ++ lib/fluent/plugin/in_systemd.rb | 14 ++ lib/fluent/plugin/systemd/entry_mutator.rb | 14 ++ test/helper.rb | 14 ++ test/plugin/systemd/test_entry_mutator.rb | 14 ++ test/plugin/test_filter_systemd_entry.rb | 14 ++ test/plugin/test_in_systemd.rb | 14 ++ 10 files changed, 293 insertions(+), 18 deletions(-) diff --git a/LICENCE b/LICENCE index 0c221ff..130c4c6 100644 --- a/LICENCE +++ b/LICENCE @@ -1,25 +1,202 @@ -Copyright (c) 2015 Edward Robinson + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + 1. Definitions. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015-2018 Edward Robinson + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 9f41322..a3c9666 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ For systems with systemd installed you can run the tests against your installed ## Licence -[MIT](LICENCE) +[Apache-2.0](LICENCE) ## Contributions diff --git a/fluent-plugin-systemd.gemspec b/fluent-plugin-systemd.gemspec index 9718b3c..f49a439 100644 --- a/fluent-plugin-systemd.gemspec +++ b/fluent-plugin-systemd.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |spec| spec.summary = 'Input plugin to read from systemd journal.' spec.description = 'This is a fluentd input plugin. It reads logs from the systemd journal.' spec.homepage = 'https://github.com/reevoo/fluent-plugin-systemd' - spec.license = 'MIT' + spec.license = 'Apache-2.0' spec.files = Dir['lib/**/**.rb', 'README.md', 'LICENCE'] spec.require_paths = ['lib'] diff --git a/lib/fluent/plugin/filter_systemd_entry.rb b/lib/fluent/plugin/filter_systemd_entry.rb index 7840d6b..de23719 100644 --- a/lib/fluent/plugin/filter_systemd_entry.rb +++ b/lib/fluent/plugin/filter_systemd_entry.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require 'fluent/plugin/filter' require 'fluent/plugin/systemd/entry_mutator' diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb index 43e69f4..2843535 100644 --- a/lib/fluent/plugin/in_systemd.rb +++ b/lib/fluent/plugin/in_systemd.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require 'systemd/journal' require 'fluent/plugin/input' require 'fluent/plugin/systemd/entry_mutator' diff --git a/lib/fluent/plugin/systemd/entry_mutator.rb b/lib/fluent/plugin/systemd/entry_mutator.rb index 7c73100..42ead23 100644 --- a/lib/fluent/plugin/systemd/entry_mutator.rb +++ b/lib/fluent/plugin/systemd/entry_mutator.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require 'fluent/config/error' module Fluent diff --git a/test/helper.rb b/test/helper.rb index 3a136f8..a1439d6 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require 'test/unit' require 'fluent/test' require 'fluent/test/helpers' diff --git a/test/plugin/systemd/test_entry_mutator.rb b/test/plugin/systemd/test_entry_mutator.rb index d8248b0..d0fe836 100644 --- a/test/plugin/systemd/test_entry_mutator.rb +++ b/test/plugin/systemd/test_entry_mutator.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require_relative '../../helper' require 'json' require 'systemd/journal' diff --git a/test/plugin/test_filter_systemd_entry.rb b/test/plugin/test_filter_systemd_entry.rb index e9f74f9..9c1c545 100644 --- a/test/plugin/test_filter_systemd_entry.rb +++ b/test/plugin/test_filter_systemd_entry.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require_relative '../helper' require_relative './systemd/test_entry_mutator' require 'fluent/test/driver/filter' diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb index b00a517..14d5d8c 100644 --- a/test/plugin/test_in_systemd.rb +++ b/test/plugin/test_in_systemd.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# Copyright 2015-2018 Edward Robinson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + require_relative '../helper' require_relative './systemd/test_entry_mutator' require 'tempfile' From 78340f5235321337b9eb47f68da8a97660453517 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 3 May 2018 21:55:18 +0100 Subject: [PATCH 11/16] Third prerelease --- fluent-plugin-systemd.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-systemd.gemspec b/fluent-plugin-systemd.gemspec index f49a439..0d15f43 100644 --- a/fluent-plugin-systemd.gemspec +++ b/fluent-plugin-systemd.gemspec @@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = 'fluent-plugin-systemd' - spec.version = '1.0.0.rc2' + spec.version = '1.0.0.rc3' spec.authors = ['Ed Robinson'] spec.email = ['edward-robinson@cookpad.com'] From ff4b3283d9cf91376b97754b4dae61cac99f0449 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Wed, 9 May 2018 10:53:09 +0100 Subject: [PATCH 12/16] Adds documentation about upgrading to 1.0.0 --- README.md | 25 +++++------------------ docs/upgrading.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 docs/upgrading.md diff --git a/README.md b/README.md index a3c9666..4b5dd8c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Overview -**systemd** input plugin reads logs from the systemd journal +**systemd** input plugin reads logs from the systemd journal **systemd** filter plugin allows for basic manipulation of systemd journal entries ## Support @@ -35,6 +35,10 @@ or td-agent-gem install fluent-plugin-systemd -v 0.3.1 +## Upgrading + +If you are upgrading to version 1.0 from a previous version of this plugin take a look at the [upgrade documentation](docs/upgrading.md). A number of deprecated config options were removed so you might need to update your configuration. + ## Input Plugin Configuration @@ -76,34 +80,15 @@ reading all logs from the journal. See [matching details](docs/Matching-Details.md) for a more exhaustive description of this property and how to use it. -**`pos_file`** - -_This parameter is deprecated and will be removed in favour of storage in v1.0._ - - -Path to pos file, stores the journald cursor. File is created if does not exist. - **`storage`** Configuration for a [storage plugin](http://docs.fluentd.org/v0.14/articles/storage-plugin-overview) used to store the journald cursor. -_Upgrading from `pos_file`_ - -If `pos_file` is specified in addition to a storage plugin with persistent set to true, the cursor will be -copied from the `pos_file` on startup, and the old `pos_file` removed. - **`read_from_head`** If true reads all available journal from head, otherwise starts reading from tail, ignored if pos file exists (and is valid). Defaults to false. -**`strip_underscores`** - -_This parameter is deprecated and will be removed in favour of entry in v1.0._ - -If true strips underscores from the beginning of systemd field names. -May be useful if outputting to kibana, as underscore prefixed fields are unindexed there. - **`entry`** Optional configuration for an embeded systemd entry filter. See the [Filter Plugin Configuration](#filter-plugin-configuration) for config reference. diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 0000000..d0eda59 --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,52 @@ +# Upgrading + +## To Version 1.0 + +Version 1.0 removes a number of configuration options that had been deprecated by previous versions of the plugin. This was done to reduce the size of the code base and make maintenance simpler. + +If you have been paying attention to (and fixing) the deprecation warnings introduced by previous versions of the plugin then there is nothing for you to do. If you have not already done so it is recommended to first upgrade to version `0.3.1` and fix any warnings before trying version `1.0.0` or above. + +Version 1.0 of fluent-plugin-systemd only supports fluentd 0.14.11 and above (including fluentd 1.0+), if you are using tdagent you need to be using version 3 or above. + +### `pos_file` + +Previous versions of the plugin used the `pos_file` config value to specify a file that the position or cursor from the systemd journal would be written to. This was replaced by a generic fluentd storage block that allows much more flexibility in how the cursor is persisted. Take a look at the [fluentd documentation](https://docs.fluentd.org/v1.0/articles/storage-section) to find out more about this. + +Before you upgrade to 1.0 you should migrate `pos_file` to a storage block. + +``` +pos_file /var/log/journald.pos +``` + +could be rewritten as + +``` + + @type local + persistent true + path /var/log/journald_pos.json + +``` + +If you want to update this configuration without skipping any entries if you supply the `pos_file` and a storage block at the same time version `0.3.1` will copy the cursor from the path given in `pos_file` to the given storage. + +### `strip_underscores` + +The legacy `strip_underscores` method is removed in version `1.0.0` and above. The same functionality can be achieved by setting the `fields_strip_underscores` on an entry block. The entry block allows many more options for mutating journal entries. + +``` +strip_underscores true +``` + +should be rewritten as + +``` + + fields_strip_underscores true + +``` + +### `filters` + +In version 1.0.0 the `filters` parameter was renamed as `matches` in order to more closely align the plugin with the names used in the systemd documentation. `filters` is deprecated and will be removed in a future version. Other than renaming the parameter no changes have been made to it's structure or operation. + From 4d0b405423397995a3b64181824847549e904852 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Wed, 9 May 2018 11:20:21 +0100 Subject: [PATCH 13/16] Fixup field_map examples fixes #63 --- README.md | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4b5dd8c..62d23ca 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ ## Overview -**systemd** input plugin reads logs from the systemd journal -**systemd** filter plugin allows for basic manipulation of systemd journal entries +* **systemd** input plugin to read logs from the systemd journal +* **systemd** filter plugin for basic manipulation of systemd journal entries ## Support @@ -53,7 +53,6 @@ If you are upgrading to version 1.0 from a previous version of this plugin take path kube-proxy.pos - field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"} fields_strip_underscores true fields_lowercase true @@ -69,7 +68,7 @@ Path to the systemd journal, defaults to `/var/log/journal` **`filters`** -_This parameter name is deprecated and should be renamed to `matches`_ +_This parameter name is depreciated and should be renamed to `matches`_ **`matches`** @@ -105,13 +104,17 @@ For an example of a full working setup including the plugin, [take a look at](ht ## Filter Plugin Configuration - - @type systemd_entry - field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"} - field_map_strict false - fields_lowercase true - fields_strip_underscores true - +``` + + @type systemd_entry + field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"} + field_map_strict false + fields_lowercase true + fields_strip_underscores true + +``` + +_Note that the following configurations can be embedded in a systemd source block, within an entry block, you only need to use a filter directly for more complicated workflows._ **`field_map`** @@ -136,7 +139,7 @@ If true, lowercase all non-mapped fields. Defaults to false. If true, strip leading underscores from all non-mapped fields. Defaults to false. -### Example +### Filter Example Given a systemd journal source entry: ``` @@ -165,10 +168,10 @@ The resulting entry using the above sample configuration: > ### When I look at fluentd logs, everything looks fine but no journal logs are read -This is commonly caused when the user running fluentd does not have enough permisions +This is commonly caused when the user running fluentd does not have the correct permissions to read the systemd journal. -Acording to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html): +According to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html): > Journal files are, by default, owned and readable by the "systemd-journal" system group but are not writable. Adding a user to this group thus enables her/him to read the journal files. @@ -182,7 +185,7 @@ To run the tests with docker on several distros simply run `rake` For systems with systemd installed you can run the tests against your installed libsystemd with `rake test` -## Licence +## License [Apache-2.0](LICENCE) From 9000ded94c3c2a911be7cddb7d9913d359b10077 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 10 May 2018 11:54:32 +0100 Subject: [PATCH 14/16] Adds documentation about multi-line Fixes #59 --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62d23ca..19ceabd 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Join the #plugin-systemd channel on the [Fluentd Slack](http://slack.fluentd.org | 0.0.x | ~> 0.12.0 | 2 | >= 1.9 | * The 0.x.x series is developed from this branch (master) -* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x) +* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is maintained on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x) * The next version is currently under development on the [1.0.0 branch](https://github.com/reevoo/fluent-plugin-systemd/tree/1.0.0) it's progress is tracked [here](https://github.com/reevoo/fluent-plugin-systemd/issues/53) ## Installation @@ -86,11 +86,11 @@ Configuration for a [storage plugin](http://docs.fluentd.org/v0.14/articles/stor **`read_from_head`** If true reads all available journal from head, otherwise starts reading from tail, - ignored if pos file exists (and is valid). Defaults to false. + ignored if cursor exists in storage (and is valid). Defaults to false. **`entry`** -Optional configuration for an embeded systemd entry filter. See the [Filter Plugin Configuration](#filter-plugin-configuration) for config reference. +Optional configuration for an embedded systemd entry filter. See the [Filter Plugin Configuration](#filter-plugin-configuration) for config reference. **`tag`** @@ -166,7 +166,7 @@ The resulting entry using the above sample configuration: ## Common Issues -> ### When I look at fluentd logs, everything looks fine but no journal logs are read +> ### When I look at fluentd logs, everything looks fine but no journal logs are read ? This is commonly caused when the user running fluentd does not have the correct permissions to read the systemd journal. @@ -174,6 +174,16 @@ to read the systemd journal. According to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html): > Journal files are, by default, owned and readable by the "systemd-journal" system group but are not writable. Adding a user to this group thus enables her/him to read the journal files. +> ### How can I deal with multi-line logs ? + +Ideally you want to ensure that your logs are saved to the systemd journal as a single entry regardless of how many lines they span. + +It is possible for applications to naively support this (but only if they have tight integration with systemd it seems) see: https://github.com/systemd/systemd/issues/5188. + +Typically you would not be able to this, so another way is to configure your logger to replace newline characters with something else. See this blog post for an example configuring a Java logging library to do this https://fabianlee.org/2018/03/09/java-collapsing-multiline-stack-traces-into-a-single-log-event-using-spring-backed-by-logback-or-log4j2/ + +Another strategy would be to use a plugin like [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) to combine multi line logs into a single event, this is more tricky though because you need to be able to identify the first and last lines of a multi line message with a regex. + ## Dependencies From 91faf3c6fd61c6465ad7cd09220169b616e81c81 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 10 May 2018 12:10:19 +0100 Subject: [PATCH 15/16] Adds some information about using in a Docker container Fixes #61 Signed-off-by: Ed Robinson --- README.md | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 19ceabd..d41db3b 100644 --- a/README.md +++ b/README.md @@ -98,9 +98,6 @@ _Required_ A tag that will be added to events generated by this input. -### Example - -For an example of a full working setup including the plugin, [take a look at](https://github.com/assemblyline/fluentd) ## Filter Plugin Configuration @@ -184,11 +181,38 @@ Typically you would not be able to this, so another way is to configure your log Another strategy would be to use a plugin like [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) to combine multi line logs into a single event, this is more tricky though because you need to be able to identify the first and last lines of a multi line message with a regex. +> ### How can I use this plugin inside of a docker container ? + +* Install the [systemd dependencies](#dependencies) if required +* You can use an [offical fluentd docker](https://github.com/fluent/fluentd-docker-image) image as a base, (choose the debian based version, as alpine linux doesn't support systemd). +* Bind mount `/var/log/journal` into your container. + +### Example + +For an example of a full working setup including the plugin, take a look at [the fluentd kubernetes daemonset](https://github.com/fluent/fluentd-kubernetes-daemonset) ## Dependencies This plugin depends on libsystemd +On Debian or Ubuntu you might need to install the libsystemd0 package: + +``` +apt-get install libsystemd0 +``` + +On CentOS or RHEL you might need to install the systemd package: + +``` +yum install -y systemd +``` + +If you want to do this in a CentOS docker image you might first need to remove the `fakesystemd` package. + +``` +yum remove -y fakesystemd +``` + ## Running the tests To run the tests with docker on several distros simply run `rake` From 81549813adad456c34d11043fd89788465742d58 Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 10 May 2018 15:14:43 +0100 Subject: [PATCH 16/16] =?UTF-8?q?Version=201.0.0=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ed Robinson --- README.md | 9 ++++----- docs/{Matching-Details.md => matching.md} | 0 fluent-plugin-systemd.gemspec | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) rename docs/{Matching-Details.md => matching.md} (100%) diff --git a/README.md b/README.md index d41db3b..d0ccb3d 100644 --- a/README.md +++ b/README.md @@ -21,19 +21,18 @@ Join the #plugin-systemd channel on the [Fluentd Slack](http://slack.fluentd.org | > 0.1.0 | >= 0.14.11, < 2 | 3 | >= 2.1 | | 0.0.x | ~> 0.12.0 | 2 | >= 1.9 | -* The 0.x.x series is developed from this branch (master) +* The 1.x.x series is developed from this branch (master) * The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is maintained on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x) -* The next version is currently under development on the [1.0.0 branch](https://github.com/reevoo/fluent-plugin-systemd/tree/1.0.0) it's progress is tracked [here](https://github.com/reevoo/fluent-plugin-systemd/issues/53) ## Installation Simply use RubyGems: - gem install fluent-plugin-systemd -v 0.3.1 + gem install fluent-plugin-systemd -v 1.0.0 or - td-agent-gem install fluent-plugin-systemd -v 0.3.1 + td-agent-gem install fluent-plugin-systemd -v 1.0.0 ## Upgrading @@ -76,7 +75,7 @@ Expects an array of hashes defining desired matches to filter the log messages with. When this property is not specified, this plugin will default to reading all logs from the journal. -See [matching details](docs/Matching-Details.md) for a more exhaustive +See [matching details](docs/matching.md) for a more exhaustive description of this property and how to use it. **`storage`** diff --git a/docs/Matching-Details.md b/docs/matching.md similarity index 100% rename from docs/Matching-Details.md rename to docs/matching.md diff --git a/fluent-plugin-systemd.gemspec b/fluent-plugin-systemd.gemspec index 0d15f43..1bc3d39 100644 --- a/fluent-plugin-systemd.gemspec +++ b/fluent-plugin-systemd.gemspec @@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = 'fluent-plugin-systemd' - spec.version = '1.0.0.rc3' + spec.version = '1.0.0' spec.authors = ['Ed Robinson'] spec.email = ['edward-robinson@cookpad.com']