From eeeb38db77da24aa73dd551f6134af5836955df7 Mon Sep 17 00:00:00 2001 From: novaksam Date: Thu, 23 May 2019 14:52:50 -0500 Subject: [PATCH 01/10] Added in parameter enabled timeframe support These changes allow a user to specify a timeframe, in either seconds or milliseconds, to poll an API that requires timestamps as a parameter. Example: https://docs.umbrella.com/umbrella-api/docs/security-activity-report --- lib/logstash/inputs/http_poller.rb | 62 ++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 9b814b1..4b776ff 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -5,6 +5,7 @@ require "socket" # for Socket.gethostname require "manticore" require "rufus/scheduler" +require "date" class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base include LogStash::PluginMixins::HttpClient @@ -35,6 +36,23 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base # hash of metadata. config :metadata_target, :validate => :string, :default => '@metadata' + # The name of a variable to use in string replacement for time based calls in the past + # making available as a variable to work around hard-coded string substition issues + config :time_back_buffer_string, :validate => :string, :default => 'time_back_buffer' + + # The amount of time in seconds to poll backwards + config :time_back_buffer, :validate => :number + + # The name of a variable to use in string replacement for time based calls in the past + # making available as a variable to work around hard-coded string substition issues + config :time_forward_buffer_string, :validate => :string, :default => 'time_forward_buffer' + + # The amount of time in seconds to poll forwards + config :time_forward_buffer, :validate => :number, :default => 0 + + # get the timeformat, to support seconds and milliseconds + config :time_format, :validate => ['seconds','milliseconds'], :default => 'seconds' + public Schedule_types = %w(cron every at in) def register @@ -147,8 +165,52 @@ def run_once(queue) private def request_async(queue, name, request) @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request) + @logger.debug? && @logger.debug("Forward Buffer", :buffer => @time_back_buffer) + @logger.debug? && @logger.debug("Backward Buffer", :buffer => @time_forward_buffer) + @logger.debug? && @logger.debug("Time Format", :format => @time_format) + # Grab the current time started = Time.now + # this needs to be a DateTime to deal with subtractions + currenttime = DateTime.now + @logger.debug? && @logger.debug("Current Time", :currenttime => currenttime) + # If we have the @time_back_buffer set, we modify the URL with a calculated timestamp + back_buffer = @time_back_buffer + forward_buffer = @time_forward_buffer + + # To support multiple formats + # https://apidock.com/ruby/DateTime/strftime + if @time_format == "seconds" + time_format_code = '%s' + elsif @time_format == "milliseconds" + time_format_code = '%Q' + end + + # Deal with buffers going backwards + if @time_back_buffer && @time_back_buffer > 0 + # Rational is fractions, and the second number is the number of seconds in a day + # Datetime is the time since the unix epoch, and it works using rational numbers + # https://stackoverflow.com/a/10056201 has more info + buffer = currenttime - Rational(back_buffer,86400) + @logger.debug? && @logger.debug("Back Buffer", :buffer => buffer) + request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + #test1 = request[1].gsub(/#{time_back_buffer_string}/,back_buffer.strftime('%Q')) + #test2 = request[1].gsub(@time_back_buffer_string,back_buffer.strftime('%Q')) + end + + # deal with forward buffers, if we need to + # We can tolerate a zero here because it would indicate 'now' + if @time_forward_buffer && @time_forward_buffer >= 0 + # Rational is fractions, and the second number is the number of seconds in a day + # Datetime is the time since the unix epoch, and it works using rational numbers + # https://stackoverflow.com/a/10056201 has more info + buffer = currenttime + Rational(forward_buffer,86400) + @logger.debug? && @logger.debug("Forward Buffer", :buffer => buffer) + request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + #test1 = request[1].gsub(/#{time_back_buffer_string}/,back_buffer.strftime('%Q')) + #test2 = request[1].gsub(@time_back_buffer_string,back_buffer.strftime('%Q')) + end + method, *request_opts = request client.async.send(method, *request_opts). on_success {|response| handle_success(queue, name, request, response, Time.now - started)}. From 70fcf02735cfce151483e3a8ec1818460f29b563 Mon Sep 17 00:00:00 2001 From: novaksam Date: Fri, 24 May 2019 10:51:36 -0500 Subject: [PATCH 02/10] Allow more flexibility with time formatting Instead of hard-coded Seconds/Milliseconds, I've opted to allow the time_format to be user set to any Ruby supported format, which should allow much more flexibility for URL formatting. --- lib/logstash/inputs/http_poller.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 4b776ff..57590f6 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -51,7 +51,12 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base config :time_forward_buffer, :validate => :number, :default => 0 # get the timeformat, to support seconds and milliseconds - config :time_format, :validate => ['seconds','milliseconds'], :default => 'seconds' + # Common time formats and codes + # %FT%R - 2007-11-19T08:37 Calendar date and local time (extended) + # %FT%T%:z - 2007-11-19T08:37:48-06:00 Date and time of day for calendar date (extended) + # %s - Number of seconds since 1970-01-01 00:00:00 UTC. + # %Q - Number of milliseconds since 1970-01-01 00:00:00 UTC. + config :time_format, :validate => :string public Schedule_types = %w(cron every at in) @@ -180,10 +185,8 @@ def request_async(queue, name, request) # To support multiple formats # https://apidock.com/ruby/DateTime/strftime - if @time_format == "seconds" - time_format_code = '%s' - elsif @time_format == "milliseconds" - time_format_code = '%Q' + if @time_format + time_format_code = @time_format end # Deal with buffers going backwards @@ -194,8 +197,6 @@ def request_async(queue, name, request) buffer = currenttime - Rational(back_buffer,86400) @logger.debug? && @logger.debug("Back Buffer", :buffer => buffer) request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) - #test1 = request[1].gsub(/#{time_back_buffer_string}/,back_buffer.strftime('%Q')) - #test2 = request[1].gsub(@time_back_buffer_string,back_buffer.strftime('%Q')) end # deal with forward buffers, if we need to @@ -207,8 +208,6 @@ def request_async(queue, name, request) buffer = currenttime + Rational(forward_buffer,86400) @logger.debug? && @logger.debug("Forward Buffer", :buffer => buffer) request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) - #test1 = request[1].gsub(/#{time_back_buffer_string}/,back_buffer.strftime('%Q')) - #test2 = request[1].gsub(@time_back_buffer_string,back_buffer.strftime('%Q')) end method, *request_opts = request From cf6fc5650587cf3bcbc4272e6aa1af745ae50d93 Mon Sep 17 00:00:00 2001 From: novaksam Date: Fri, 24 May 2019 11:11:49 -0500 Subject: [PATCH 03/10] Forgot to default the parameter Forgot to set a default to the parameter, resulting in a nul pointer. My bad. --- lib/logstash/inputs/http_poller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 57590f6..1da7379 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -56,7 +56,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base # %FT%T%:z - 2007-11-19T08:37:48-06:00 Date and time of day for calendar date (extended) # %s - Number of seconds since 1970-01-01 00:00:00 UTC. # %Q - Number of milliseconds since 1970-01-01 00:00:00 UTC. - config :time_format, :validate => :string + config :time_format, :validate => :string, :default => '%s' public Schedule_types = %w(cron every at in) From 3b8c18df70e6ec688b03fe5424a609cf9038699b Mon Sep 17 00:00:00 2001 From: novaksam Date: Fri, 24 May 2019 12:27:16 -0500 Subject: [PATCH 04/10] Add parameters in to index.asciidoc --- docs/index.asciidoc | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index c06e573..19b67da 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -117,6 +117,11 @@ This plugin supports the following configuration options plus the <> |<>|Yes | <> |<>|No | <> |<>|No +| <>|No +| <>|No +| <>|No +| <>|No +| <>|No | <> |a valid filesystem path|No | <> |<>|No | <> |<>|No @@ -324,6 +329,51 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s` Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event. +[id="plugins-{type}s-{plugin}-time_back_buffer_string"] +===== time_back_buffer_string + + * Value type is <> + * Default value is `time_back_buffer` + +The text to replace in the URL with a formatted time string that represents a time in the past + +[id="plugins-{type}s-{plugin}-time_back_buffer"] +===== time_back_buffer + + * Value type is <> + * There is no default value for this setting + +The time in seconds to set the 'time_back_buffer' backwards + +[id="plugins-{type}s-{plugin}-time_forward_buffer_string"] +===== time_forward_buffer_string + + * Value type is <> + * Default value is `time_forward_buffer` + +The text to replace in the URL with a formatted time string that represents a time in the future + +[id="plugins-{type}s-{plugin}-time_forward_buffer"] +===== time_forward_buffer + + * Value type is <> + * Default value is `0` + +The time in seconds to set the 'time_forward_buffer' forwards + +[id="plugins-{type}s-{plugin}-time_format"] +===== time_format + + * Value type is <> + * Default value is `'%s'` + +The string representing the ruby DateTime strftime output format. +Examples include: + 1. %FT%R - 2007-11-19T08:37 Calendar date and local time (extended) + 2. %FT%T%:z - 2007-11-19T08:37:48-06:00 Date and time of day for calendar date (extended) + 3. %s - Number of seconds since 1970-01-01 00:00:00 UTC. + 4. %Q - Number of milliseconds since 1970-01-01 00:00:00 UTC. + [id="plugins-{type}s-{plugin}-truststore"] ===== `truststore` From 2b3a22d4d2ce84754ddf16f9be28d986657a4554 Mon Sep 17 00:00:00 2001 From: novaksam Date: Fri, 24 May 2019 12:27:27 -0500 Subject: [PATCH 05/10] Debugging typo --- lib/logstash/inputs/http_poller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 1da7379..eeafef4 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -170,8 +170,8 @@ def run_once(queue) private def request_async(queue, name, request) @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request) - @logger.debug? && @logger.debug("Forward Buffer", :buffer => @time_back_buffer) - @logger.debug? && @logger.debug("Backward Buffer", :buffer => @time_forward_buffer) + @logger.debug? && @logger.debug("Forward Buffer", :buffer => @time_forward_buffer) + @logger.debug? && @logger.debug("Backward Buffer", :buffer => @time_back_buffer) @logger.debug? && @logger.debug("Time Format", :format => @time_format) # Grab the current time started = Time.now From 9d15d1d2b9300880c0f1811b93278c81bd8bbc96 Mon Sep 17 00:00:00 2001 From: novaksam Date: Fri, 24 May 2019 12:29:45 -0500 Subject: [PATCH 06/10] Minor formatting issues in index.asciidoc --- docs/index.asciidoc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 19b67da..1b4a3f6 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -117,11 +117,11 @@ This plugin supports the following configuration options plus the <> |<>|Yes | <> |<>|No | <> |<>|No -| <>|No -| <>|No -| <>|No -| <>|No -| <>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No | <> |a valid filesystem path|No | <> |<>|No | <> |<>|No @@ -330,7 +330,7 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s` Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event. [id="plugins-{type}s-{plugin}-time_back_buffer_string"] -===== time_back_buffer_string +===== `time_back_buffer_string` * Value type is <> * Default value is `time_back_buffer` @@ -338,7 +338,7 @@ Define the target field for placing the received data. If this setting is omitte The text to replace in the URL with a formatted time string that represents a time in the past [id="plugins-{type}s-{plugin}-time_back_buffer"] -===== time_back_buffer +===== `time_back_buffer` * Value type is <> * There is no default value for this setting @@ -346,7 +346,7 @@ The text to replace in the URL with a formatted time string that represents a ti The time in seconds to set the 'time_back_buffer' backwards [id="plugins-{type}s-{plugin}-time_forward_buffer_string"] -===== time_forward_buffer_string +===== `time_forward_buffer_string` * Value type is <> * Default value is `time_forward_buffer` @@ -354,7 +354,7 @@ The time in seconds to set the 'time_back_buffer' backwards The text to replace in the URL with a formatted time string that represents a time in the future [id="plugins-{type}s-{plugin}-time_forward_buffer"] -===== time_forward_buffer +===== `time_forward_buffer` * Value type is <> * Default value is `0` @@ -362,7 +362,7 @@ The text to replace in the URL with a formatted time string that represents a ti The time in seconds to set the 'time_forward_buffer' forwards [id="plugins-{type}s-{plugin}-time_format"] -===== time_format +===== `time_format` * Value type is <> * Default value is `'%s'` From 56246c113c44a20ab5d30a32b0e13b34164fec7b Mon Sep 17 00:00:00 2001 From: novaksam Date: Fri, 24 May 2019 15:44:18 -0500 Subject: [PATCH 07/10] Allow support for multiple URLs and pollers I found out that, seemingly, in the case of having several instances of http_poller or multiple URLs, the modifcations to the URL wasn't happening, so now there is support for dealing with the request arrays, along with a micro optimization of only running gsub what the text we're replacing is actually there. --- lib/logstash/inputs/http_poller.rb | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index eeafef4..650f0cd 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -196,7 +196,18 @@ def request_async(queue, name, request) # https://stackoverflow.com/a/10056201 has more info buffer = currenttime - Rational(back_buffer,86400) @logger.debug? && @logger.debug("Back Buffer", :buffer => buffer) - request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + request.each_with_index do |entry, i| + # We need to verify we're working with a string, otherwise we run in to method not found errors + if request[i].is_a? String + # And lets only modify strings that actually include our text + if request[i].include?("#{time_back_buffer_string}") + # Originally request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[i]) + request[i] = request[i].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i]) + end + end + end end # deal with forward buffers, if we need to @@ -207,7 +218,18 @@ def request_async(queue, name, request) # https://stackoverflow.com/a/10056201 has more info buffer = currenttime + Rational(forward_buffer,86400) @logger.debug? && @logger.debug("Forward Buffer", :buffer => buffer) - request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + request.each_with_index do |entry, i| + # We need to verify we're working with a string, otherwise we run in to method not found errors + if request[i].is_a? String + # And lets only modify strings that actually include our text + if request[i].include?("#{time_forward_buffer_string}") + # Originally request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + @logger.debug? && @logger.debug("URL timestamp - forwards - pre:", :url => request[i]) + request[i] = request[i].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i]) + end + end + end end method, *request_opts = request From b8989142eaf1e814751f498b63098934e7ada546 Mon Sep 17 00:00:00 2001 From: novaksam Date: Tue, 28 May 2019 08:49:18 -0500 Subject: [PATCH 08/10] Always re-set the URL string to contain our buffer character I observed some situations where the URL was being set only at the first run, resulting in a failure to obtain new results as time moved forward. I added in some code to both failure and success to make sure the URL was reset after every run. --- lib/logstash/inputs/http_poller.rb | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 650f0cd..13d5879 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -204,7 +204,10 @@ def request_async(queue, name, request) # Originally request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[i]) request[i] = request[i].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # Store the timestamp as a variable to swap it back after the URL has been fetched + @buffer_time_back = buffer.strftime(time_format_code) @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i]) + @logger.debug? && @logger.debug("URL timestamp - backwards - string:", :time => buffer_time_back) end end end @@ -226,7 +229,10 @@ def request_async(queue, name, request) # Originally request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) @logger.debug? && @logger.debug("URL timestamp - forwards - pre:", :url => request[i]) request[i] = request[i].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # Store the timestamp as a variable to swap it back after the URL has been fetched + @buffer_time_forward = buffer.strftime(time_format_code) @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i]) + @logger.debug? && @logger.debug("URL timestamp - forwards - string:", :time => buffer_time_forward) end end end @@ -234,9 +240,30 @@ def request_async(queue, name, request) method, *request_opts = request client.async.send(method, *request_opts). - on_success {|response| handle_success(queue, name, request, response, Time.now - started)}. + on_success {|response| + @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1]) + handle_success(queue, name, request, response, Time.now - started) + # If either of out buffers were set, replace the contents of the URL back to our string + # It has been observed that the URL wasn't being set back, hence this workaround + if @buffer_time_back + request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string) + end + if @buffer_time_forward + request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string) + end + @logger.debug? && @logger.debug("URL timestamp - success - post:", :url => request[1])}. on_failure {|exception| + @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1]) handle_failure(queue, name, request, exception, Time.now - started) + # If either of out buffers were set, replace the contents of the URL back to our string + # It has been observed that the URL wasn't being set back, hence this workaround + if @buffer_time_back + request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string) + end + if @buffer_time_forward + request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string) + end + @logger.debug? && @logger.debug("URL timestamp - failure - post:", :url => request[1]) } end From 4fb341b2fc164db1d9973c650ff2b89cb8ac2812 Mon Sep 17 00:00:00 2001 From: novaksam Date: Tue, 28 May 2019 08:51:39 -0500 Subject: [PATCH 09/10] Formatting fixes Adjusting whitespace, nothing to see here. --- lib/logstash/inputs/http_poller.rb | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index 13d5879..f4b53d3 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -208,9 +208,9 @@ def request_async(queue, name, request) @buffer_time_back = buffer.strftime(time_format_code) @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i]) @logger.debug? && @logger.debug("URL timestamp - backwards - string:", :time => buffer_time_back) - end - end - end + end + end + end end # deal with forward buffers, if we need to @@ -233,34 +233,34 @@ def request_async(queue, name, request) @buffer_time_forward = buffer.strftime(time_format_code) @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i]) @logger.debug? && @logger.debug("URL timestamp - forwards - string:", :time => buffer_time_forward) - end - end - end + end + end + end end method, *request_opts = request client.async.send(method, *request_opts). on_success {|response| - @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1]) + @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1]) handle_success(queue, name, request, response, Time.now - started) - # If either of out buffers were set, replace the contents of the URL back to our string - # It has been observed that the URL wasn't being set back, hence this workaround - if @buffer_time_back + # If either of out buffers were set, replace the contents of the URL back to our string + # It has been observed that the URL wasn't being set back, hence this workaround + if @buffer_time_back request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string) end - if @buffer_time_forward + if @buffer_time_forward request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string) end @logger.debug? && @logger.debug("URL timestamp - success - post:", :url => request[1])}. on_failure {|exception| - @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1]) + @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1]) handle_failure(queue, name, request, exception, Time.now - started) - # If either of out buffers were set, replace the contents of the URL back to our string - # It has been observed that the URL wasn't being set back, hence this workaround - if @buffer_time_back + # If either of out buffers were set, replace the contents of the URL back to our string + # It has been observed that the URL wasn't being set back, hence this workaround + if @buffer_time_back request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string) end - if @buffer_time_forward + if @buffer_time_forward request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string) end @logger.debug? && @logger.debug("URL timestamp - failure - post:", :url => request[1]) From 005cc4c564cb10ddc6706064968ab3ffe27c62ef Mon Sep 17 00:00:00 2001 From: novaksam Date: Tue, 28 May 2019 11:45:17 -0500 Subject: [PATCH 10/10] Removing loop After doing some additional testing, it turns out I don't need a loop, but I left it in as a comment for reference. --- lib/logstash/inputs/http_poller.rb | 76 ++++++++++++++++++------------ 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/lib/logstash/inputs/http_poller.rb b/lib/logstash/inputs/http_poller.rb index f4b53d3..c9e3da4 100644 --- a/lib/logstash/inputs/http_poller.rb +++ b/lib/logstash/inputs/http_poller.rb @@ -196,21 +196,30 @@ def request_async(queue, name, request) # https://stackoverflow.com/a/10056201 has more info buffer = currenttime - Rational(back_buffer,86400) @logger.debug? && @logger.debug("Back Buffer", :buffer => buffer) - request.each_with_index do |entry, i| - # We need to verify we're working with a string, otherwise we run in to method not found errors - if request[i].is_a? String - # And lets only modify strings that actually include our text - if request[i].include?("#{time_back_buffer_string}") - # Originally request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) - @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[i]) - request[i] = request[i].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) - # Store the timestamp as a variable to swap it back after the URL has been fetched - @buffer_time_back = buffer.strftime(time_format_code) - @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i]) - @logger.debug? && @logger.debug("URL timestamp - backwards - string:", :time => buffer_time_back) - end - end + # Turns out I don't need to handle arrays, as each request is processed separately + # but I'll keep the loop here, for reference + if request[1].include?("#{time_back_buffer_string}") + @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[1]) + request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # Store the timestamp as a variable to swap it back after the URL has been fetched + @buffer_time_back = buffer.strftime(time_format_code) + @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[1]) end + #request.each_with_index do |entry, i| + # # We need to verify we're working with a string, otherwise we run in to method not found errors + # if request[i].is_a? String + # # And lets only modify strings that actually include our text + # if request[i].include?("#{time_back_buffer_string}") + # # Originally request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[i]) + # request[i] = request[i].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code)) + # # Store the timestamp as a variable to swap it back after the URL has been fetched + # @buffer_time_back = buffer.strftime(time_format_code) + # @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i]) + # @logger.debug? && @logger.debug("URL timestamp - backwards - string:", :time => buffer_time_back) + # end + # end + #end end # deal with forward buffers, if we need to @@ -221,27 +230,34 @@ def request_async(queue, name, request) # https://stackoverflow.com/a/10056201 has more info buffer = currenttime + Rational(forward_buffer,86400) @logger.debug? && @logger.debug("Forward Buffer", :buffer => buffer) - request.each_with_index do |entry, i| - # We need to verify we're working with a string, otherwise we run in to method not found errors - if request[i].is_a? String - # And lets only modify strings that actually include our text - if request[i].include?("#{time_forward_buffer_string}") - # Originally request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) - @logger.debug? && @logger.debug("URL timestamp - forwards - pre:", :url => request[i]) - request[i] = request[i].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) - # Store the timestamp as a variable to swap it back after the URL has been fetched - @buffer_time_forward = buffer.strftime(time_format_code) - @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i]) - @logger.debug? && @logger.debug("URL timestamp - forwards - string:", :time => buffer_time_forward) - end - end + if request[1].include?("#{time_forward_buffer_string}") + @logger.debug? && @logger.debug("URL timestamp - forward - pre:", :url => request[1]) + request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # Store the timestamp as a variable to swap it back after the URL has been fetched + @buffer_time_forward = buffer.strftime(time_format_code) + @logger.debug? && @logger.debug("URL timestamp - forward - post:", :url => request[1]) end + # request.each_with_index do |entry, i| + # # We need to verify we're working with a string, otherwise we run in to method not found errors + # if request[i].is_a? String + # # And lets only modify strings that actually include our text + # if request[i].include?("#{time_forward_buffer_string}") + # # Originally request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # @logger.debug? && @logger.debug("URL timestamp - forwards - pre:", :url => request[i]) + # request[i] = request[i].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code)) + # # Store the timestamp as a variable to swap it back after the URL has been fetched + # @buffer_time_forward = buffer.strftime(time_format_code) + # @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i]) + # @logger.debug? && @logger.debug("URL timestamp - forwards - string:", :time => buffer_time_forward) + # end + # end + #end end method, *request_opts = request client.async.send(method, *request_opts). on_success {|response| - @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1]) + @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1]) handle_success(queue, name, request, response, Time.now - started) # If either of out buffers were set, replace the contents of the URL back to our string # It has been observed that the URL wasn't being set back, hence this workaround @@ -253,7 +269,7 @@ def request_async(queue, name, request) end @logger.debug? && @logger.debug("URL timestamp - success - post:", :url => request[1])}. on_failure {|exception| - @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1]) + @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1]) handle_failure(queue, name, request, exception, Time.now - started) # If either of out buffers were set, replace the contents of the URL back to our string # It has been observed that the URL wasn't being set back, hence this workaround