diff --git a/LICENCE b/LICENCE
index 0c221ff..130c4c6 100644
--- a/LICENCE
+++ b/LICENCE
@@ -1,25 +1,202 @@
-Copyright (c) 2015 Edward Robinson
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+ 1. Definitions.
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2015-2018 Edward Robinson
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/README.md b/README.md
index 73a5698..d0ccb3d 100644
--- a/README.md
+++ b/README.md
@@ -4,8 +4,8 @@
## Overview
-**systemd** input plugin reads logs from the systemd journal
-**systemd** filter plugin allows for basic manipulation of systemd journal entries
+* **systemd** input plugin to read logs from the systemd journal
+* **systemd** filter plugin for basic manipulation of systemd journal entries
## Support
@@ -21,19 +21,22 @@ Join the #plugin-systemd channel on the [Fluentd Slack](http://slack.fluentd.org
| > 0.1.0 | >= 0.14.11, < 2 | 3 | >= 2.1 |
| 0.0.x | ~> 0.12.0 | 2 | >= 1.9 |
-* The 0.x.x series is developed from this branch (master)
-* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x)
-* The next version is currently under development on the [1.0.0 branch](https://github.com/reevoo/fluent-plugin-systemd/tree/1.0.0) it's progress is tracked [here](https://github.com/reevoo/fluent-plugin-systemd/issues/53)
+* The 1.x.x series is developed from this branch (master)
+* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is maintained on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x)
## Installation
Simply use RubyGems:
- gem install fluent-plugin-systemd -v 0.3.1
+ gem install fluent-plugin-systemd -v 1.0.0
or
- td-agent-gem install fluent-plugin-systemd -v 0.3.1
+ td-agent-gem install fluent-plugin-systemd -v 1.0.0
+
+## Upgrading
+
+If you are upgrading to version 1.0 from a previous version of this plugin take a look at the [upgrade documentation](docs/upgrading.md). A number of deprecated config options were removed so you might need to update your configuration.
## Input Plugin Configuration
@@ -41,7 +44,7 @@ or
@type systemd
tag kube-proxy
path /var/log/journal
- filters [{ "_SYSTEMD_UNIT": "kube-proxy.service" }]
+ matches [{ "_SYSTEMD_UNIT": "kube-proxy.service" }]
read_from_head true
@type local
@@ -49,7 +52,6 @@ or
path kube-proxy.pos
- field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
fields_strip_underscores true
fields_lowercase true
@@ -65,44 +67,29 @@ Path to the systemd journal, defaults to `/var/log/journal`
**`filters`**
-Expects an array of hashes defining desired filters to apply to all log
-messages. When this property is not specified, this plugin will default to
-having no filters specified.
-
-See [filtering details](docs/Filtering-Details.md) for a more exhaustive
-description of this property and how to use it.
-
-**`pos_file`**
+_This parameter name is depreciated and should be renamed to `matches`_
-_This parameter is deprecated and will be removed in favour of storage in v1.0._
+**`matches`**
+Expects an array of hashes defining desired matches to filter the log
+messages with. When this property is not specified, this plugin will default to
+reading all logs from the journal.
-Path to pos file, stores the journald cursor. File is created if does not exist.
+See [matching details](docs/matching.md) for a more exhaustive
+description of this property and how to use it.
**`storage`**
Configuration for a [storage plugin](http://docs.fluentd.org/v0.14/articles/storage-plugin-overview) used to store the journald cursor.
-_Upgrading from `pos_file`_
-
-If `pos_file` is specified in addition to a storage plugin with persistent set to true, the cursor will be
-copied from the `pos_file` on startup, and the old `pos_file` removed.
-
**`read_from_head`**
If true reads all available journal from head, otherwise starts reading from tail,
- ignored if pos file exists (and is valid). Defaults to false.
-
-**`strip_underscores`**
-
-_This parameter is deprecated and will be removed in favour of entry in v1.0._
-
-If true strips underscores from the beginning of systemd field names.
-May be useful if outputting to kibana, as underscore prefixed fields are unindexed there.
+ ignored if cursor exists in storage (and is valid). Defaults to false.
**`entry`**
-Optional configuration for an embeded systemd entry filter. See the [Filter Plugin Configuration](#filter-plugin-configuration) for config reference.
+Optional configuration for an embedded systemd entry filter. See the [Filter Plugin Configuration](#filter-plugin-configuration) for config reference.
**`tag`**
@@ -110,19 +97,20 @@ _Required_
A tag that will be added to events generated by this input.
-### Example
-
-For an example of a full working setup including the plugin, [take a look at](https://github.com/assemblyline/fluentd)
## Filter Plugin Configuration
-
- @type systemd_entry
- field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
- field_map_strict false
- fields_lowercase true
- fields_strip_underscores true
-
+```
+
+ @type systemd_entry
+ field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
+ field_map_strict false
+ fields_lowercase true
+ fields_strip_underscores true
+
+```
+
+_Note that the following configurations can be embedded in a systemd source block, within an entry block, you only need to use a filter directly for more complicated workflows._
**`field_map`**
@@ -147,7 +135,7 @@ If true, lowercase all non-mapped fields. Defaults to false.
If true, strip leading underscores from all non-mapped fields. Defaults to false.
-### Example
+### Filter Example
Given a systemd journal source entry:
```
@@ -174,28 +162,65 @@ The resulting entry using the above sample configuration:
## Common Issues
-> ### When I look at fluentd logs, everything looks fine but no journal logs are read
+> ### When I look at fluentd logs, everything looks fine but no journal logs are read ?
-This is commonly caused when the user running fluentd does not have enough permisions
+This is commonly caused when the user running fluentd does not have the correct permissions
to read the systemd journal.
-Acording to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html):
+According to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html):
> Journal files are, by default, owned and readable by the "systemd-journal" system group but are not writable. Adding a user to this group thus enables her/him to read the journal files.
+> ### How can I deal with multi-line logs ?
+
+Ideally you want to ensure that your logs are saved to the systemd journal as a single entry regardless of how many lines they span.
+
+It is possible for applications to naively support this (but only if they have tight integration with systemd it seems) see: https://github.com/systemd/systemd/issues/5188.
+
+Typically you would not be able to this, so another way is to configure your logger to replace newline characters with something else. See this blog post for an example configuring a Java logging library to do this https://fabianlee.org/2018/03/09/java-collapsing-multiline-stack-traces-into-a-single-log-event-using-spring-backed-by-logback-or-log4j2/
+
+Another strategy would be to use a plugin like [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) to combine multi line logs into a single event, this is more tricky though because you need to be able to identify the first and last lines of a multi line message with a regex.
+
+> ### How can I use this plugin inside of a docker container ?
+
+* Install the [systemd dependencies](#dependencies) if required
+* You can use an [offical fluentd docker](https://github.com/fluent/fluentd-docker-image) image as a base, (choose the debian based version, as alpine linux doesn't support systemd).
+* Bind mount `/var/log/journal` into your container.
+
+### Example
+
+For an example of a full working setup including the plugin, take a look at [the fluentd kubernetes daemonset](https://github.com/fluent/fluentd-kubernetes-daemonset)
## Dependencies
This plugin depends on libsystemd
+On Debian or Ubuntu you might need to install the libsystemd0 package:
+
+```
+apt-get install libsystemd0
+```
+
+On CentOS or RHEL you might need to install the systemd package:
+
+```
+yum install -y systemd
+```
+
+If you want to do this in a CentOS docker image you might first need to remove the `fakesystemd` package.
+
+```
+yum remove -y fakesystemd
+```
+
## Running the tests
To run the tests with docker on several distros simply run `rake`
For systems with systemd installed you can run the tests against your installed libsystemd with `rake test`
-## Licence
+## License
-[MIT](LICENCE)
+[Apache-2.0](LICENCE)
## Contributions
diff --git a/Rakefile b/Rakefile
index 9b6a0a2..433f63b 100644
--- a/Rakefile
+++ b/Rakefile
@@ -22,12 +22,7 @@ namespace :docker do
distros.each do |distro|
task distro do
puts "testing on #{distro}"
- begin
- FileUtils.cp("test/docker/Dockerfile.#{distro}", 'Dockerfile')
- sh 'docker build .'
- ensure
- FileUtils.rm('Dockerfile')
- end
+ sh "docker build . -f test/docker/Dockerfile.#{distro}"
end
end
end
diff --git a/docs/Filtering-Details.md b/docs/matching.md
similarity index 75%
rename from docs/Filtering-Details.md
rename to docs/matching.md
index a2a12ec..ac72e29 100644
--- a/docs/Filtering-Details.md
+++ b/docs/matching.md
@@ -1,35 +1,38 @@
-# Filtering Details
+# Matching Details
## Overview
-This application takes an array of hashes passed to the `filters` parameter
+This application takes an array of hashes passed to the `matches` parameter
within a `systemd` typed source definition in your `fluent.conf` configuration
file and then parses them into a format understood by `libsystemd`'s journal
API. The basis behind what `libsystemd`'s API expects can be found documented in
the `journalctl` [man
page](https://www.freedesktop.org/software/systemd/man/journalctl.html).
+The result of this is that only logs which match the defined set of matching
+rules will be further processed.
+
## Usage Information
-In order to utilize this plugin's filtering capabilities, you will need to
+In order to utilize this plugin's matching capabilities, you will need to
understand how this plugin transforms the passed array of hashes into a format
that is understood by `libsystemd`.
The best way to describe this process is probably by example. The following
sub-sections lists out various scenarios that you might wish to perform with
-this plugin's filtering mechanism and describes both how to configure them,
+this plugin's matching mechanism and describes both how to configure them,
while also mapping them to examples from the `journalctl` [man
page](https://www.freedesktop.org/software/systemd/man/journalctl.html).
### No Filters
-You can leave the `filters` property out altogether, or include a `filters`
-property with an empty array (as shown below) to specify that no filtering
+You can leave the `matches` property out altogether, or include a `matches`
+property with an empty array (as shown below) to specify that no matching
should occur.
- filters []
+ matches []
-Which matches this part of the `journalctl` man page:
+Which coincides with this part of the `journalctl` man page:
> Without arguments, all collected logs are shown unfiltered:
>
@@ -37,13 +40,13 @@ Which matches this part of the `journalctl` man page:
### Single Filter
-You can pass a single hash map to the `filters` array with a single key/value
-pair specified to filter out all log entries that do not match the given
-field/value combination.
+You can pass a single hash map to the `matches` array with a single key/value
+pair specified to only process log entries that match the given field/value
+combination.
For example:
- filters [{"_SYSTEMD_UNIT": "avahi-daemon.service"}]
+ matches [{"_SYSTEMD_UNIT": "avahi-daemon.service"}]
Which coincides with this part of the the `journalctl` man page:
@@ -54,9 +57,9 @@ Which coincides with this part of the the `journalctl` man page:
### Multi-Field Filters
-You can pass a single hash map to the `filters` array with multiple key/value
-pairs to filter out all log entries that do not match the combination of all of
-the specified key/value combinations.
+You can pass a single hash map to the `matches` array with multiple key/value
+pairs to only process log entries that match the combination of all of the
+specified key/value combinations.
The passed key/value pairs are treated as a logical `AND`, such that all of the
pairs must be true in order to allow further processing of the current log
@@ -64,7 +67,7 @@ entry.
For Example:
- filters [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}]
+ matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}]
Which coincides with this part of the the `journalctl` man page:
@@ -74,15 +77,15 @@ Which coincides with this part of the the `journalctl` man page:
> `journalctl _SYSTEMD_UNIT=avahi-daemon.service _PID=28097`
You can also perform a logical `OR` by splitting key/value pairs across multiple
-hashes passed to the `filters` array like so:
+hashes passed to the `matches` array like so:
- filters [{"_SYSTEMD_UNIT": "avahi-daemon.service"}, {"_PID": 28097}]
+ matches [{"_SYSTEMD_UNIT": "avahi-daemon.service"}, {"_PID": 28097}]
You can combine both `AND` and `OR` combinations together; using a single hash
map to define conditions that `AND` together and using multiple hash maps to
define conditions that `OR` together like so:
- filters [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}]
+ matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}]
This can be expressed in psuedo-code like so:
@@ -105,7 +108,7 @@ Fields with arrays as values are treated as a logical `OR` statement.
For example:
- filters [{"_SYSTEMD_UNIT": ["avahi-daemon.service", "dbus.service"]}]
+ matches [{"_SYSTEMD_UNIT": ["avahi-daemon.service", "dbus.service"]}]
Which coincides with this part of the `journalctl` man page:
@@ -119,7 +122,7 @@ particularly helpful when you want to create aggregate logic
For example:
- filters [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}]
+ matches [{"_SYSTEMD_UNIT": "avahi-daemon.service", "_PID": 28097}, {"_SYSTEMD_UNIT": "dbus.service"}]
This can be expressed in psuedo-code like so:
diff --git a/docs/upgrading.md b/docs/upgrading.md
new file mode 100644
index 0000000..d0eda59
--- /dev/null
+++ b/docs/upgrading.md
@@ -0,0 +1,52 @@
+# Upgrading
+
+## To Version 1.0
+
+Version 1.0 removes a number of configuration options that had been deprecated by previous versions of the plugin. This was done to reduce the size of the code base and make maintenance simpler.
+
+If you have been paying attention to (and fixing) the deprecation warnings introduced by previous versions of the plugin then there is nothing for you to do. If you have not already done so it is recommended to first upgrade to version `0.3.1` and fix any warnings before trying version `1.0.0` or above.
+
+Version 1.0 of fluent-plugin-systemd only supports fluentd 0.14.11 and above (including fluentd 1.0+), if you are using tdagent you need to be using version 3 or above.
+
+### `pos_file`
+
+Previous versions of the plugin used the `pos_file` config value to specify a file that the position or cursor from the systemd journal would be written to. This was replaced by a generic fluentd storage block that allows much more flexibility in how the cursor is persisted. Take a look at the [fluentd documentation](https://docs.fluentd.org/v1.0/articles/storage-section) to find out more about this.
+
+Before you upgrade to 1.0 you should migrate `pos_file` to a storage block.
+
+```
+pos_file /var/log/journald.pos
+```
+
+could be rewritten as
+
+```
+
+ @type local
+ persistent true
+ path /var/log/journald_pos.json
+
+```
+
+If you want to update this configuration without skipping any entries if you supply the `pos_file` and a storage block at the same time version `0.3.1` will copy the cursor from the path given in `pos_file` to the given storage.
+
+### `strip_underscores`
+
+The legacy `strip_underscores` method is removed in version `1.0.0` and above. The same functionality can be achieved by setting the `fields_strip_underscores` on an entry block. The entry block allows many more options for mutating journal entries.
+
+```
+strip_underscores true
+```
+
+should be rewritten as
+
+```
+
+ fields_strip_underscores true
+
+```
+
+### `filters`
+
+In version 1.0.0 the `filters` parameter was renamed as `matches` in order to more closely align the plugin with the names used in the systemd documentation. `filters` is deprecated and will be removed in a future version. Other than renaming the parameter no changes have been made to it's structure or operation.
+
diff --git a/fluent-plugin-systemd.gemspec b/fluent-plugin-systemd.gemspec
index c347870..1bc3d39 100644
--- a/fluent-plugin-systemd.gemspec
+++ b/fluent-plugin-systemd.gemspec
@@ -6,14 +6,14 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
Gem::Specification.new do |spec|
spec.name = 'fluent-plugin-systemd'
- spec.version = '0.3.1'
+ spec.version = '1.0.0'
spec.authors = ['Ed Robinson']
spec.email = ['edward-robinson@cookpad.com']
spec.summary = 'Input plugin to read from systemd journal.'
spec.description = 'This is a fluentd input plugin. It reads logs from the systemd journal.'
spec.homepage = 'https://github.com/reevoo/fluent-plugin-systemd'
- spec.license = 'MIT'
+ spec.license = 'Apache-2.0'
spec.files = Dir['lib/**/**.rb', 'README.md', 'LICENCE']
spec.require_paths = ['lib']
diff --git a/lib/fluent/plugin/filter_systemd_entry.rb b/lib/fluent/plugin/filter_systemd_entry.rb
index 7840d6b..de23719 100644
--- a/lib/fluent/plugin/filter_systemd_entry.rb
+++ b/lib/fluent/plugin/filter_systemd_entry.rb
@@ -1,5 +1,19 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require 'fluent/plugin/filter'
require 'fluent/plugin/systemd/entry_mutator'
diff --git a/lib/fluent/plugin/in_systemd.rb b/lib/fluent/plugin/in_systemd.rb
index 000f153..2843535 100644
--- a/lib/fluent/plugin/in_systemd.rb
+++ b/lib/fluent/plugin/in_systemd.rb
@@ -1,14 +1,27 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require 'systemd/journal'
require 'fluent/plugin/input'
-require 'fluent/plugin/systemd/pos_writer'
require 'fluent/plugin/systemd/entry_mutator'
module Fluent
module Plugin
# Fluentd plugin for reading from the systemd journal
- class SystemdInput < Input # rubocop:disable Metrics/ClassLength
+ class SystemdInput < Input
Fluent::Plugin.register_input('systemd', self)
helpers :timer, :storage
@@ -16,11 +29,9 @@ class SystemdInput < Input # rubocop:disable Metrics/ClassLength
DEFAULT_STORAGE_TYPE = 'local'
config_param :path, :string, default: '/var/log/journal'
- config_param :filters, :array, default: []
- config_param :pos_file, :string, default: nil, deprecated: "Use section with `persistent: true' instead"
+ config_param :filters, :array, default: [], deprecated: 'filters has been renamed as matches'
+ config_param :matches, :array, default: nil
config_param :read_from_head, :bool, default: false
- config_param :strip_underscores, :bool, default: false, deprecated: 'Use section or `systemd_entry` ' \
- 'filter plugin instead'
config_param :tag, :string
config_section :storage do
@@ -39,25 +50,16 @@ class SystemdInput < Input # rubocop:disable Metrics/ClassLength
def configure(conf)
super
@journal = nil
- @pos_storage = PosWriter.new(@pos_file, storage_create(usage: 'positions'))
- # legacy strip_underscores backwards compatibility (legacy takes
- # precedence and is mutually exclusive with the entry block)
- mut_opts = @strip_underscores ? { fields_strip_underscores: true } : @entry_opts.to_h
- @mutator = SystemdEntryMutator.new(**mut_opts)
+ @pos_storage = storage_create(usage: 'positions')
+ @mutator = SystemdEntryMutator.new(**@entry_opts.to_h)
@mutator.warnings.each { |warning| log.warn(warning) }
end
def start
super
- @pos_storage.start
timer_execute(:in_systemd_emit_worker, 1, &method(:run))
end
- def shutdown
- @pos_storage.shutdown
- super
- end
-
private
def init_journal
@@ -67,7 +69,7 @@ def init_journal
# make sure initial call to wait doesn't return :invalidate
# see https://github.com/ledbettj/systemd-journal/issues/70
@journal.wait(0)
- @journal.filter(*@filters)
+ @journal.filter(*(@matches || @filters))
seek
true
rescue Systemd::JournalError => e
@@ -80,7 +82,7 @@ def seek
seek_to(cursor || read_from)
rescue Systemd::JournalError
log.warn(
- "Could not seek to cursor #{cursor} found in pos file: #{@pos_storage.path}, " \
+ "Could not seek to cursor #{cursor} found in position file: #{@pos_storage.path}, " \
"falling back to reading from #{read_from}"
)
seek_to(read_from)
@@ -128,16 +130,15 @@ def formatted(entry)
@mutator.run(entry)
end
- def watch
- while @journal.move_next
- begin
- yield @journal.current_entry
- rescue Systemd::JournalError => e
- log.warn("Error Parsing Journal: #{e.class}: #{e.message}")
- next
- end
- @pos_storage.put(:journal, @journal.cursor)
- end
+ def watch(&block)
+ yield_current_entry(&block) while @journal.move_next
+ end
+
+ def yield_current_entry
+ yield @journal.current_entry
+ @pos_storage.put(:journal, @journal.cursor)
+ rescue Systemd::JournalError => e
+ log.warn("Error reading from Journal: #{e.class}: #{e.message}")
end
end
end
diff --git a/lib/fluent/plugin/systemd/entry_mutator.rb b/lib/fluent/plugin/systemd/entry_mutator.rb
index 7c73100..42ead23 100644
--- a/lib/fluent/plugin/systemd/entry_mutator.rb
+++ b/lib/fluent/plugin/systemd/entry_mutator.rb
@@ -1,5 +1,19 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require 'fluent/config/error'
module Fluent
diff --git a/lib/fluent/plugin/systemd/pos_writer.rb b/lib/fluent/plugin/systemd/pos_writer.rb
deleted file mode 100644
index db09027..0000000
--- a/lib/fluent/plugin/systemd/pos_writer.rb
+++ /dev/null
@@ -1,99 +0,0 @@
-# frozen_string_literal: true
-
-require 'fluent/plugin/input'
-
-module Fluent
- module Plugin
- class SystemdInput < Input
- # This is used to write the systemd cursor to the configured storage
- # We do this periodicly in a thread so as to not contend on resources
- # that might be needed for more important tasks.
- #
- # When signaled to shutdown we ensure that the most recent cursor
- # has been written.
- #
- # If fluentd stops runnning without cleanly shutting down PosWriter
- # the cursor could be up to 1 second stale
- class PosWriter
- def initialize(pos_file, storage)
- @path = pos_file
- @lock = Mutex.new
- @storage = storage
- @cursor = nil
- @written_cursor = nil
- setup
- end
-
- def get(key)
- @storage ? @storage.get(key) : @cursor
- end
-
- def put(key, cursor)
- return @storage.put(key, cursor) if @storage
- @lock.synchronize { @cursor = cursor }
- end
-
- def path
- @path || @storage.path
- end
-
- def start
- return unless @path
- @running = true
- @thread = Thread.new(&method(:work))
- end
-
- def shutdown
- return unless @path
- @running = false
- @thread.join
- write_pos
- end
-
- private
-
- def setup
- if @storage.persistent
- migrate_to_storage
- elsif @path
- @cursor = read_legacy_pos if legacy_file?
- @storage = nil
- end
- end
-
- def legacy_file?
- @path && File.exist?(@path)
- end
-
- def read_legacy_pos
- IO.read(@path).chomp
- end
-
- def migrate_to_storage
- return unless legacy_file?
- @storage.put(:journal, read_legacy_pos)
- File.delete(@path)
- @path = nil
- end
-
- def work
- while @running
- write_pos
- sleep 1
- end
- end
-
- def write_pos
- @lock.synchronize do
- if @written_cursor != @cursor
- file = File.open(@path, 'w+', 0o644)
- file.print @cursor
- file.close
- @written_cursor = @cursor
- end
- end
- end
- end
- end
- end
-end
diff --git a/test/docker/Dockerfile.tdagent-centos b/test/docker/Dockerfile.tdagent-centos
index bab1d4c..8810304 100644
--- a/test/docker/Dockerfile.tdagent-centos
+++ b/test/docker/Dockerfile.tdagent-centos
@@ -1,8 +1,8 @@
-FROM centos:7.1.1503
+FROM centos:7
RUN yum remove -y fakesystemd \
&& rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent \
- && printf "[treasuredata]\nname=TreasureData\nbaseurl=http://packages.treasuredata.com/2/redhat/\$releasever/\$basearch\ngpgcheck=1\ngpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent\n" > /etc/yum.repos.d/td.repo \
+ && printf "[treasuredata]\nname=TreasureData\nbaseurl=http://packages.treasuredata.com/3/redhat/\$releasever/\$basearch\ngpgcheck=1\ngpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent\n" > /etc/yum.repos.d/td.repo \
&& yum install -y td-agent make gcc-c++ systemd
ENV PATH /opt/td-agent/embedded/bin/:$PATH
@@ -12,4 +12,4 @@ COPY Gemfile ./
COPY fluent-plugin-systemd.gemspec ./
RUN bundle install
COPY . .
-RUN rake test TESTOPTS="-v"
+RUN bundle exec rake test TESTOPTS="-v"
diff --git a/test/docker/Dockerfile.tdagent-ubuntu b/test/docker/Dockerfile.tdagent-ubuntu
index f03144a..47e016c 100644
--- a/test/docker/Dockerfile.tdagent-ubuntu
+++ b/test/docker/Dockerfile.tdagent-ubuntu
@@ -7,7 +7,7 @@ RUN apt-get update -q \
ca-certificates \
libsystemd0 \
&& curl https://packages.treasuredata.com/GPG-KEY-td-agent | apt-key add - \
- && echo "deb http://packages.treasuredata.com/2/ubuntu/xenial/ xenial contrib" > /etc/apt/sources.list.d/treasure-data.list \
+ && echo "deb http://packages.treasuredata.com/3/ubuntu/xenial/ xenial contrib" > /etc/apt/sources.list.d/treasure-data.list \
&& apt-get update \
&& apt-get install -y td-agent \
&& apt-get clean \
@@ -22,4 +22,4 @@ COPY Gemfile ./
COPY fluent-plugin-systemd.gemspec ./
RUN bundle check || bundle install
COPY . .
-RUN rake test TESTOPTS="-v"
+RUN bundle exec rake test TESTOPTS="-v"
diff --git a/test/helper.rb b/test/helper.rb
index 3a136f8..a1439d6 100644
--- a/test/helper.rb
+++ b/test/helper.rb
@@ -1,5 +1,19 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require 'test/unit'
require 'fluent/test'
require 'fluent/test/helpers'
diff --git a/test/plugin/systemd/test_entry_mutator.rb b/test/plugin/systemd/test_entry_mutator.rb
index d8248b0..d0fe836 100644
--- a/test/plugin/systemd/test_entry_mutator.rb
+++ b/test/plugin/systemd/test_entry_mutator.rb
@@ -1,5 +1,19 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require_relative '../../helper'
require 'json'
require 'systemd/journal'
diff --git a/test/plugin/systemd/test_pos_writer.rb b/test/plugin/systemd/test_pos_writer.rb
deleted file mode 100644
index 338e0ff..0000000
--- a/test/plugin/systemd/test_pos_writer.rb
+++ /dev/null
@@ -1,157 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../helper'
-require 'tempfile'
-require 'fluent/plugin/systemd/pos_writer'
-
-class SystemdInputTest < Test::Unit::TestCase
- class FakeStorage
- def initialize(options)
- @persistent = options[:persistent]
- @store = {}
- end
-
- attr_reader :persistent
-
- def put(key, value)
- @store[key] = value
- end
-
- def get(key)
- @store[key]
- end
- end
-
- def storage(options = {})
- FakeStorage.new(options)
- end
-
- def test_reading_the_cursor_when_file_exists
- pos_file = Tempfile.new('foo.pos')
- pos_file.write('cursor_value')
- pos_file.close
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(pos_file.path, storage)
- assert_equal pos_writer.get(:journal), 'cursor_value'
- pos_file.unlink
- end
-
- def test_reading_the_cursor_when_file_does_not_exist_yet
- dir = Dir.mktmpdir('posdir')
- path = "#{dir}/foo.pos"
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage)
- assert_equal pos_writer.get(:journal), nil
- FileUtils.rm_rf dir
- end
-
- def test_reading_the_cusor_when_the_path_is_nil
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(nil, storage)
- assert_equal pos_writer.get(:journal), nil
- pos_writer.put(:journal, 'a_cursor')
- assert_equal pos_writer.get(:journal), 'a_cursor'
- end
-
- def test_writing_the_cursor_when_file_does_not_exist_yet
- dir = Dir.mktmpdir('posdir')
- path = "#{dir}/foo.pos"
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage)
- pos_writer.start
- pos_writer.put(:journal, 'this is the cursor')
- assert_equal pos_writer.get(:journal), 'this is the cursor'
- sleep 1
- assert_equal File.read(path), 'this is the cursor'
- FileUtils.rm_rf dir
- end
-
- def test_file_permission_when_file_does_not_exist_yet
- dir = Dir.mktmpdir('posdir')
- path = "#{dir}/foo.pos"
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage)
- pos_writer.start
- pos_writer.put(:journal, 'this is the cursor')
- sleep 1
- assert_equal format('%o', File::Stat.new(path).mode)[-4, 4], '0644' # rubocop:disable Style/FormatStringToken
- FileUtils.rm_rf dir
- end
-
- def test_writing_the_cursor_when_the_writer_is_shutdown
- dir = Dir.mktmpdir('posdir')
- path = "#{dir}/foo.pos"
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage)
- pos_writer.start
- pos_writer.put(:journal, 'this is the cursor')
- pos_writer.shutdown
- assert_equal File.read(path), 'this is the cursor'
- FileUtils.rm_rf dir
- end
-
- def test_writing_the_cursor_when_the_file_exists
- pos_file = Tempfile.new('foo.pos')
- pos_file.write('cursor_value')
- pos_file.close
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(pos_file.path, storage)
- assert_equal pos_writer.get(:journal), 'cursor_value'
- pos_writer.start
- pos_writer.put(:journal, 'this is the cursor')
- sleep 1
- assert_equal File.read(pos_file.path), 'this is the cursor'
- pos_file.unlink
- end
-
- def test_writing_and_then_reading_the_pos_roundtrip
- dir = Dir.mktmpdir('posdir')
- path = "#{dir}/foo.pos"
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage)
- pos_writer.start
- pos_writer.put(:journal, 'this is the cursor')
- pos_writer.shutdown
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(path, storage)
- assert_equal pos_writer.get(:journal), 'this is the cursor'
- FileUtils.rm_rf dir
- end
-
- def test_upgrading_from_pos_writer_to_storage
- store = storage(persistent: true)
- pos_file = Tempfile.new('foo.pos')
- pos_file.write('cursor_value')
- pos_file.close
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(pos_file.path, store)
-
- # It removes the old file
- assert !File.exist?(pos_file.path)
-
- # it copies the value to the store
- assert_equal store.get(:journal), 'cursor_value'
-
- # it uses the store
- pos_writer.put(:journal, 'new_value')
- assert_equal store.get(:journal), 'new_value'
-
- # start and shutdown should be noops
- pos_writer.put(:journal, 'another_value')
- assert_nil pos_writer.start
- assert_nil pos_writer.shutdown
- assert_equal pos_writer.get(:journal), 'another_value'
- assert_equal store.get(:journal), 'another_value'
- end
-
- def test_when_the_old_pos_writer_file_does_not_exist
- store = storage(persistent: true)
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new('not_a_real_path_to_a_file', store)
- assert_nil store.get(:journal)
-
- # it works
- pos_writer.put(:journal, 'new_value')
- assert_equal store.get(:journal), 'new_value'
- assert_equal pos_writer.get(:journal), 'new_value'
- end
-
- def test_when_no_pos_file_path_given
- # uses storage even if not persistent
- store = storage(persistent: false)
- pos_writer = Fluent::Plugin::SystemdInput::PosWriter.new(nil, store)
-
- pos_writer.put(:journal, 'new_value')
- assert_equal store.get(:journal), 'new_value'
- assert_equal pos_writer.get(:journal), 'new_value'
- end
-end
diff --git a/test/plugin/test_filter_systemd_entry.rb b/test/plugin/test_filter_systemd_entry.rb
index e9f74f9..9c1c545 100644
--- a/test/plugin/test_filter_systemd_entry.rb
+++ b/test/plugin/test_filter_systemd_entry.rb
@@ -1,5 +1,19 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require_relative '../helper'
require_relative './systemd/test_entry_mutator'
require 'fluent/test/driver/filter'
diff --git a/test/plugin/test_in_systemd.rb b/test/plugin/test_in_systemd.rb
index dee4747..14d5d8c 100644
--- a/test/plugin/test_in_systemd.rb
+++ b/test/plugin/test_in_systemd.rb
@@ -1,5 +1,19 @@
# frozen_string_literal: true
+# Copyright 2015-2018 Edward Robinson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
require_relative '../helper'
require_relative './systemd/test_entry_mutator'
require 'tempfile'
@@ -59,28 +73,17 @@ def setup
path test/fixture
)
- @badmsg_config = %(
- tag test
- path test/fixture/corrupt
- read_from_head true
- )
+ @storage_path = File.join(Dir.mktmpdir('pos_dir'), 'storage.json')
- # deprecated
- @strip_config = base_config + %(
- strip_underscores true
+ @storage_config = @base_config + %(
+
+ @type local
+ persistent true
+ path #{storage_path}
+
)
- pos_dir = Dir.mktmpdir('posdir')
-
- @pos_path = "#{pos_dir}/foo.pos"
-
- @pos_config = base_config + %(
- pos_file #{@pos_path}
- )
-
- @storage_path = File.join(pos_dir.to_s, 'storage.json')
-
- @head_config = @pos_config + %(
+ @head_config = @storage_config + %(
read_from_head true
)
@@ -88,7 +91,11 @@ def setup
filters [{ "_SYSTEMD_UNIT": "systemd-journald.service" }]
)
- @tail_config = @pos_config + %(
+ @matches_config = @head_config + %(
+ matches [{ "_SYSTEMD_UNIT": "systemd-journald.service" }]
+ )
+
+ @tail_config = @storage_config + %(
read_from_head false
)
@@ -96,16 +103,30 @@ def setup
tag test
path test/not_a_real_path
)
+
+ @corrupt_entries_config = %(
+ tag test
+ path test/fixture/corrupt
+ read_from_head true
+ )
end
- attr_reader :journal, :base_config, :pos_path, :pos_config, :head_config,
- :filter_config, :strip_config, :tail_config, :not_present_config,
- :badmsg_config, :storage_path
+ attr_reader :journal, :base_config, :head_config,
+ :matches_config, :filter_config, :tail_config, :not_present_config,
+ :storage_path, :storage_config, :corrupt_entries_config
def create_driver(config)
Fluent::Test::Driver::Input.new(Fluent::Plugin::SystemdInput).configure(config)
end
+ def read_pos
+ JSON.parse(File.read(storage_path))['journal']
+ end
+
+ def write_pos(pos)
+ File.write(storage_path, JSON.dump(journal: pos))
+ end
+
def test_configure_requires_tag
assert_raise Fluent::ConfigError do
create_driver('')
@@ -141,35 +162,10 @@ def test_reading_from_the_journal_tail_mutate_entry(data)
assert_equal(expected, d.events)
end
- # deprecated config option for backwards compatibility
- def test_reading_from_the_journal_tail_with_strip_underscores_legacy
- d = create_driver(strip_config)
- expected = [[
- 'test',
- 1_364_519_243,
- EntryTestData::EXPECTED[:fields_strip_underscores]
- ]]
- d.run(expect_emits: 1)
- assert_equal(expected, d.events)
- end
-
def test_storage_file_is_written
- storage_config = config_element('ROOT', '', {
- 'tag' => 'test',
- 'path' => 'test/fixture',
- '@id' => 'test-01'
- }, [
- config_element('storage', '',
- '@type' => 'local',
- 'persistent' => true,
- 'path' => @storage_path)
- ])
-
d = create_driver(storage_config)
d.run(expect_emits: 1)
- storage = JSON.parse(File.read(storage_path))
- result = storage['journal']
- assert_equal result, 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=1cd;b=4737ffc504774b3ba67020bc947f1bc0;m=42f2dd;t=4d905e4cd5a92;x=25b3f86ff2774ac4'
+ assert_equal 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=1cd;b=4737ffc504774b3ba67020bc947f1bc0;m=42f2dd;t=4d905e4cd5a92;x=25b3f86ff2774ac4', read_pos
end
def test_reading_from_head
@@ -202,6 +198,7 @@ def test_backoff_on_buffer_error
d.run(expect_emits: 1)
end
+ # deprecated and replaced with matches
def test_reading_with_filters
d = create_driver(filter_config)
d.end_if do
@@ -211,10 +208,17 @@ def test_reading_with_filters
assert_equal 3, d.events.size
end
+ def test_reading_with_matches
+ d = create_driver(matches_config)
+ d.end_if do
+ d.events.size >= 3
+ end
+ d.run(timeout: 5)
+ assert_equal 3, d.events.size
+ end
+
def test_reading_from_a_pos
- file = File.open(pos_path, 'w+')
- file.print 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=13f;b=4737ffc504774b3ba67020bc947f1bc0;m=ffadd;t=4d905e49a6291;x=9a11dd9ffee96e9f'
- file.close
+ write_pos 's=add4782f78ca4b6e84aa88d34e5b4a9d;i=13f;b=4737ffc504774b3ba67020bc947f1bc0;m=ffadd;t=4d905e49a6291;x=9a11dd9ffee96e9f'
d = create_driver(head_config)
d.end_if do
d.events.size >= 142
@@ -224,9 +228,7 @@ def test_reading_from_a_pos
end
def test_reading_from_an_invalid_pos
- file = File.open(pos_path, 'w+')
- file.print 'thisisinvalid'
- file.close
+ write_pos 'thisisinvalid'
# It continues as if the pos file did not exist
d = create_driver(head_config)
@@ -236,7 +238,7 @@ def test_reading_from_an_invalid_pos
d.run(timeout: 5)
assert_equal 461, d.events.size
assert_match(
- "Could not seek to cursor thisisinvalid found in pos file: #{pos_path}, falling back to reading from head",
+ "Could not seek to cursor thisisinvalid found in position file: #{storage_path}, falling back to reading from head",
d.logs.last
)
end
@@ -259,10 +261,9 @@ def test_journal_not_present
assert_match 'Systemd::JournalError: No such file or directory retrying in 1s', d.logs.last
end
- def test_continue_on_bad_message
- d = create_driver(badmsg_config)
+ def test_reading_from_a_journal_with_corrupted_entries
+ d = create_driver(corrupt_entries_config)
d.run(expect_emits: 460)
assert_equal 460, d.events.size
- assert_equal 0, d.error_events.size
end
end