diff --git a/docs/index.asciidoc b/docs/index.asciidoc index c06e573..1b4a3f6 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -117,6 +117,11 @@ This plugin supports the following configuration options plus the <> |<>|Yes | <> |<>|No | <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No | <> |a valid filesystem path|No | <> |<>|No | <> |<>|No @@ -324,6 +329,51 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s` Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event. +[id="plugins-{type}s-{plugin}-time_back_buffer_string"] +===== `time_back_buffer_string` + + * Value type is <> + * Default value is `time_back_buffer` + +The text to replace in the URL with a formatted time string that represents a time in the past + +[id="plugins-{type}s-{plugin}-time_back_buffer"] +===== `time_back_buffer` + + * Value type is <> + * There is no default value for this setting + +The time in seconds to set the 'time_back_buffer' backwards + +[id="plugins-{type}s-{plugin}-time_forward_buffer_string"] +===== `time_forward_buffer_string` + + * Value type is <> + * Default value is `time_forward_buffer` + +The text to replace in the URL with a formatted time string that represents a time in the future + +[id="plugins-{type}s-{plugin}-time_forward_buffer"] +===== `time_forward_buffer` + + * Value type is <> + * Default value is `0` + +The time in seconds to set the 'time_forward_buffer' forwards + +[id="plugins-{type}s-{plugin}-time_format"] +===== `time_format` + + * Value type is <> + * Default value is `'%s'` + +The string representing the ruby DateTime strftime output format. +Examples include: + 1. %FT%R - 2007-11-19T08:37 Calendar date and local time (extended) + 2. %FT%T%:z - 2007-11-19T08:37:48-06:00 Date and time of day for calendar date (extended) + 3. %s - Number of seconds since 1970-01-01 00:00:00 UTC. + 4. %Q - Number of milliseconds since 1970-01-01 00:00:00 UTC. + [id="plugins-{type}s-{plugin}-truststore"] ===== `truststore` diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 9b814b1..c9e3da4 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -5,6 +5,7 @@ require "socket" # for Socket.gethostname require "manticore" require "rufus/scheduler" +require "date" class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base include LogStash::PluginMixins::HttpClient @@ -35,6 +36,28 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base # hash of metadata. config :metadata_target, :validate => :string, :default => '@metadata' + # The name of a variable to use in string replacement for time based calls in the past + # making available as a variable to work around hard-coded string substition issues + config :time_back_buffer_string, :validate => :string, :default => 'time_back_buffer' + + # The amount of time in seconds to poll backwards + config :time_back_buffer, :validate => :number + + # The name of a variable to use in string replacement for time based calls in the past + # making available as a variable to work around hard-coded string substition issues + config :time_forward_buffer_string, :validate => :string, :default => 'time_forward_buffer' + + # The amount of time in seconds to poll forwards + config :time_forward_buffer, :validate => :number, :default => 0 + + # get the timeformat, to support seconds and milliseconds + # Common time formats and codes + # %FT%R - 2007-11-19T08:37 Calendar date and local time (extended) + # %FT%T%:z - 2007-11-19T08:37:48-06:00 Date and time of day for calendar date (extended) + # %s - Number of seconds since 1970-01-01 00:00:00 UTC. + # %Q - Number of milliseconds since 1970-01-01 00:00:00 UTC. + config :time_format, :validate => :string, :default => '%s' + public Schedule_types = %w(cron every at in) def register @@ -147,13 +170,116 @@ def run_once(queue) private def request_async(queue, name, request) @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request) + @logger.debug? && @logger.debug("Forward Buffer", :buffer => @time_forward_buffer) + @logger.debug? && @logger.debug("Backward Buffer", :buffer => @time_back_buffer) + @logger.debug? && @logger.debug("Time Format", :format => @time_format) + # Grab the current time started = Time.now + # this needs to be a DateTime to deal with subtractions + currenttime = DateTime.now + @logger.debug? && @logger.debug("Current Time", :currenttime => currenttime) + # If we have the @time_back_buffer set, we modify the URL with a calculated timestamp + back_buffer = @time_back_buffer + forward_buffer = @time_forward_buffer + + # To support multiple formats + # https://apidock.com/ruby/DateTime/strftime + if @time_format + time_format_code = @time_format + end + + # Deal with buffers going backwards + if @time_back_buffer && @time_back_buffer > 0 + # Rational is fractions, and the second number is the number of seconds in a day + # Datetime is the time since the unix epoch, and it works using rational numbers + # https://stackoverflow.com/a/10056201 has more info + buffer = currenttime - Rational(back_buffer,86400) + @logger.debug? && @logger.debug("Back Buffer", :buffer => buffer) + # Turns out I don't need to handle arrays, as each request is processed separately + # but I'll keep the loop here, for reference + if request[1].include?("#{time_back_buffer_string}") + @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[1]) + request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # Store the timestamp as a variable to swap it back after the URL has been fetched + @buffer_time_back = buffer.strftime(time_format_code) + @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[1]) + end + #request.each_with_index do |entry, i| + # # We need to verify we're working with a string, otherwise we run in to method not found errors + # if request[i].is_a? String + # # And lets only modify strings that actually include our text + # if request[i].include?("#{time_back_buffer_string}") + # # Originally request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[i]) + # request[i] = request[i].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # # Store the timestamp as a variable to swap it back after the URL has been fetched + # @buffer_time_back = buffer.strftime(time_format_code) + # @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i]) + # @logger.debug? && @logger.debug("URL timestamp - backwards - string:", :time => buffer_time_back) + # end + # end + #end + end + + # deal with forward buffers, if we need to + # We can tolerate a zero here because it would indicate 'now' + if @time_forward_buffer && @time_forward_buffer >= 0 + # Rational is fractions, and the second number is the number of seconds in a day + # Datetime is the time since the unix epoch, and it works using rational numbers + # https://stackoverflow.com/a/10056201 has more info + buffer = currenttime + Rational(forward_buffer,86400) + @logger.debug? && @logger.debug("Forward Buffer", :buffer => buffer) + if request[1].include?("#{time_forward_buffer_string}") + @logger.debug? && @logger.debug("URL timestamp - forward - pre:", :url => request[1]) + request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # Store the timestamp as a variable to swap it back after the URL has been fetched + @buffer_time_forward = buffer.strftime(time_format_code) + @logger.debug? && @logger.debug("URL timestamp - forward - post:", :url => request[1]) + end + # request.each_with_index do |entry, i| + # # We need to verify we're working with a string, otherwise we run in to method not found errors + # if request[i].is_a? String + # # And lets only modify strings that actually include our text + # if request[i].include?("#{time_forward_buffer_string}") + # # Originally request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # @logger.debug? && @logger.debug("URL timestamp - forwards - pre:", :url => request[i]) + # request[i] = request[i].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # # Store the timestamp as a variable to swap it back after the URL has been fetched + # @buffer_time_forward = buffer.strftime(time_format_code) + # @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i]) + # @logger.debug? && @logger.debug("URL timestamp - forwards - string:", :time => buffer_time_forward) + # end + # end + #end + end + method, *request_opts = request client.async.send(method, *request_opts). - on_success {|response| handle_success(queue, name, request, response, Time.now - started)}. + on_success {|response| + @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1]) + handle_success(queue, name, request, response, Time.now - started) + # If either of out buffers were set, replace the contents of the URL back to our string + # It has been observed that the URL wasn't being set back, hence this workaround + if @buffer_time_back + request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string) + end + if @buffer_time_forward + request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string) + end + @logger.debug? && @logger.debug("URL timestamp - success - post:", :url => request[1])}. on_failure {|exception| + @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1]) handle_failure(queue, name, request, exception, Time.now - started) + # If either of out buffers were set, replace the contents of the URL back to our string + # It has been observed that the URL wasn't being set back, hence this workaround + if @buffer_time_back + request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string) + end + if @buffer_time_forward + request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string) + end + @logger.debug? && @logger.debug("URL timestamp - failure - post:", :url => request[1]) } end