From 1a412a3d1ae84b600e90d77976c6b3b5b164a8c9 Mon Sep 17 00:00:00 2001 From: Yuri Smirnov <tycoooon@gmail.com> Date: Fri, 28 Jun 2024 21:19:14 +0600 Subject: [PATCH] Limit number of log lines stored in memory (#44) * limit log lines * add Lamian.config.max_log_lines * update gems * bump version * add nodoc --- Changelog.md | 16 +- Gemfile | 14 + Gemfile.lock | 311 ++++++++++--------- lamian.gemspec | 14 - lib/lamian.rb | 1 + lib/lamian/config.rb | 7 +- lib/lamian/log_device.rb | 24 ++ lib/lamian/logger.rb | 3 +- lib/lamian/version.rb | 2 +- spec/lamian/log_device_spec.rb | 16 + spec/lamian/semantic_logger_appender_spec.rb | 3 +- spec/support/cool_loggers.rb | 2 +- 12 files changed, 236 insertions(+), 177 deletions(-) create mode 100644 lib/lamian/log_device.rb create mode 100644 spec/lamian/log_device_spec.rb diff --git a/Changelog.md b/Changelog.md index 541ad02..48b4a7a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,19 +1,21 @@ -# Lamian version changes (since 0.1.0) +## 1.9.0 -Update this on a pull request, under `Lamian::VERSION` -(also known as next version). If this constant would be changed without release, -I'll update it here too +* Add `max_log_lines` config option to limit number of most recent log lines stored in the log device ## 1.8.0 + * Minor dependency updates; ## 1.3.0 + * Add support for the (new sentry gem)[https://github.com/getsentry/sentry-ruby]. ## 1.2.0 + * Add `raven_log_size_limit` config option for limiting amount of data sent to sentry (defaults to `500_000`) ## 1.1.0 + * Add support for sentry and sidekiq ## 1.0.0 @@ -30,7 +32,6 @@ which ruins concept of single entry point :(. Also tied it to lamian instance * `e57e6cec` Changed rails dependency from `~> 4.2` to `>= 4.2` - ## 0.3.1 * 34ca83b5 Fixed formatting @@ -38,7 +39,6 @@ which ruins concept of single entry point :(. Also tied it to lamian instance Stabilized formatting api, which removes control sequences from loggers data. E.g. `"[23mNice, lol[0m\n"` becomes `"Nice, lol\n"` - ## 0.3.0 * `d24f895b` API update @@ -46,13 +46,13 @@ E.g. `"[23mNice, lol[0m\n"` becomes `"Nice, lol\n"` Updated API, so lamian is now forced to be used with block. It also simplified usage outside a middleware - ## 0.2.0 + * `3166517e` Added integrtation with rails Injected middleware before `ExceptionNotification`, so `ExceptionNotification` can use current log without any configuration. Also added some views - ## 0.1.0 + * `62eb8685` Made test version to check it's integration with rails application diff --git a/Gemfile b/Gemfile index 919ce9e..46fd231 100644 --- a/Gemfile +++ b/Gemfile @@ -4,3 +4,17 @@ source "https://rubygems.org" # Specify your gem's dependencies in lamian.gemspec gemspec + +gem "bundler-audit" +gem "ci-helper" +gem "launchy" +gem "pry" +gem "rake" +gem "rspec" +gem "rubocop-config-umbrellio" +gem "semantic_logger" +gem "sentry-raven" +gem "sentry-ruby" +gem "simplecov" +gem "simplecov-lcov" +gem "yard" diff --git a/Gemfile.lock b/Gemfile.lock index a47d70c..8e1ba96 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,41 +1,41 @@ PATH remote: . specs: - lamian (1.8.0) + lamian (1.9.0) rails (>= 4.2) GEM remote: https://rubygems.org/ specs: - actioncable (7.1.3) - actionpack (= 7.1.3) - activesupport (= 7.1.3) + actioncable (7.1.3.4) + actionpack (= 7.1.3.4) + activesupport (= 7.1.3.4) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.3) - actionpack (= 7.1.3) - activejob (= 7.1.3) - activerecord (= 7.1.3) - activestorage (= 7.1.3) - activesupport (= 7.1.3) + actionmailbox (7.1.3.4) + actionpack (= 7.1.3.4) + activejob (= 7.1.3.4) + activerecord (= 7.1.3.4) + activestorage (= 7.1.3.4) + activesupport (= 7.1.3.4) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.1.3) - actionpack (= 7.1.3) - actionview (= 7.1.3) - activejob (= 7.1.3) - activesupport (= 7.1.3) + actionmailer (7.1.3.4) + actionpack (= 7.1.3.4) + actionview (= 7.1.3.4) + activejob (= 7.1.3.4) + activesupport (= 7.1.3.4) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.2) - actionpack (7.1.3) - actionview (= 7.1.3) - activesupport (= 7.1.3) + actionpack (7.1.3.4) + actionview (= 7.1.3.4) + activesupport (= 7.1.3.4) nokogiri (>= 1.8.5) racc rack (>= 2.2.4) @@ -43,35 +43,35 @@ GEM rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.3) - actionpack (= 7.1.3) - activerecord (= 7.1.3) - activestorage (= 7.1.3) - activesupport (= 7.1.3) + actiontext (7.1.3.4) + actionpack (= 7.1.3.4) + activerecord (= 7.1.3.4) + activestorage (= 7.1.3.4) + activesupport (= 7.1.3.4) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.3) - activesupport (= 7.1.3) + actionview (7.1.3.4) + activesupport (= 7.1.3.4) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.3) - activesupport (= 7.1.3) + activejob (7.1.3.4) + activesupport (= 7.1.3.4) globalid (>= 0.3.6) - activemodel (7.1.3) - activesupport (= 7.1.3) - activerecord (7.1.3) - activemodel (= 7.1.3) - activesupport (= 7.1.3) + activemodel (7.1.3.4) + activesupport (= 7.1.3.4) + activerecord (7.1.3.4) + activemodel (= 7.1.3.4) + activesupport (= 7.1.3.4) timeout (>= 0.4.0) - activestorage (7.1.3) - actionpack (= 7.1.3) - activejob (= 7.1.3) - activerecord (= 7.1.3) - activesupport (= 7.1.3) + activestorage (7.1.3.4) + actionpack (= 7.1.3.4) + activejob (= 7.1.3.4) + activerecord (= 7.1.3.4) + activesupport (= 7.1.3.4) marcel (~> 1.0) - activesupport (7.1.3) + activesupport (7.1.3.4) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -81,46 +81,48 @@ GEM minitest (>= 5.1) mutex_m tzinfo (~> 2.0) - addressable (2.8.1) - public_suffix (>= 2.0.2, < 6.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) base64 (0.2.0) - bigdecimal (3.1.6) - builder (3.2.4) + bigdecimal (3.1.8) + builder (3.3.0) bundler-audit (0.9.1) bundler (>= 1.2.0, < 3) thor (~> 1.0) - ci-helper (0.5.0) - colorize (~> 0.8) - dry-inflector (~> 0.2) - umbrellio-sequel-plugins (~> 0.4) + childprocess (5.0.0) + ci-helper (0.6.0) + colorize (~> 1.1) + dry-inflector (~> 1.0) + umbrellio-sequel-plugins (~> 0.14) coderay (1.1.3) - colorize (0.8.1) - concurrent-ruby (1.2.3) + colorize (1.1.0) + concurrent-ruby (1.3.3) connection_pool (2.4.1) crass (1.0.6) date (3.3.4) - diff-lcs (1.5.0) + diff-lcs (1.5.1) docile (1.4.0) - drb (2.2.0) - ruby2_keywords - dry-inflector (0.3.0) - erubi (1.12.0) - faraday (2.6.0) - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-net_http (3.0.1) + drb (2.2.1) + dry-inflector (1.0.0) + erubi (1.13.0) + faraday (2.9.2) + faraday-net_http (>= 2.0, < 3.2) + faraday-net_http (3.1.0) + net-http globalid (1.2.1) activesupport (>= 6.1) - i18n (1.14.1) + i18n (1.14.5) concurrent-ruby (~> 1.0) io-console (0.7.2) - irb (1.11.1) - rdoc + irb (1.13.2) + rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.6.2) - launchy (2.5.0) - addressable (~> 2.7) + json (2.7.2) + language_server-protocol (3.17.0.3) + launchy (3.0.1) + addressable (~> 2.8) + childprocess (~> 5.0) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -129,61 +131,64 @@ GEM net-imap net-pop net-smtp - marcel (1.0.2) - method_source (1.0.0) + marcel (1.0.4) + method_source (1.1.0) mini_mime (1.1.5) - mini_portile2 (2.8.5) - minitest (5.21.2) + mini_portile2 (2.8.7) + minitest (5.24.0) mutex_m (0.2.0) - net-imap (0.4.9.1) + net-http (0.4.1) + uri + net-imap (0.4.14) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-smtp (0.4.0.1) + net-smtp (0.5.0) net-protocol - nio4r (2.7.0) - nokogiri (1.16.0) + nio4r (2.7.3) + nokogiri (1.16.6) mini_portile2 (~> 2.8.2) racc (~> 1.4) - nokogiri (1.16.0-x86_64-darwin) + nokogiri (1.16.6-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.0-x86_64-linux) + nokogiri (1.16.6-x86_64-linux) racc (~> 1.4) - parallel (1.22.1) - parser (3.1.2.1) + parallel (1.25.1) + parser (3.3.3.0) ast (~> 2.4.1) - pry (0.14.1) + racc + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) psych (5.1.2) stringio - public_suffix (5.0.0) - racc (1.7.3) - rack (3.0.8) + public_suffix (6.0.0) + racc (1.8.0) + rack (3.1.4) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) rack (>= 1.3) - rackup (2.0.0) + rackup (2.1.0) rack (>= 3) - webrick - rails (7.1.3) - actioncable (= 7.1.3) - actionmailbox (= 7.1.3) - actionmailer (= 7.1.3) - actionpack (= 7.1.3) - actiontext (= 7.1.3) - actionview (= 7.1.3) - activejob (= 7.1.3) - activemodel (= 7.1.3) - activerecord (= 7.1.3) - activestorage (= 7.1.3) - activesupport (= 7.1.3) + webrick (~> 1.8) + rails (7.1.3.4) + actioncable (= 7.1.3.4) + actionmailbox (= 7.1.3.4) + actionmailer (= 7.1.3.4) + actionpack (= 7.1.3.4) + actiontext (= 7.1.3.4) + actionview (= 7.1.3.4) + activejob (= 7.1.3.4) + activemodel (= 7.1.3.4) + activerecord (= 7.1.3.4) + activestorage (= 7.1.3.4) + activesupport (= 7.1.3.4) bundler (>= 1.15.0) - railties (= 7.1.3) + railties (= 7.1.3.4) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -191,100 +196,114 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.3) - actionpack (= 7.1.3) - activesupport (= 7.1.3) + railties (7.1.3.4) + actionpack (= 7.1.3.4) + activesupport (= 7.1.3.4) irb rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) rainbow (3.1.1) - rake (13.1.0) - rdoc (6.6.2) + rake (13.2.1) + rdoc (6.7.0) psych (>= 4.0.0) - regexp_parser (2.6.0) - reline (0.4.2) + regexp_parser (2.9.2) + reline (0.5.9) io-console (~> 0.5) - rexml (3.2.5) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.2) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) + rexml (3.3.1) + strscan + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.1) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.1) - rubocop (1.35.1) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + rubocop (1.63.5) json (~> 2.3) + language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.1.2.1) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.20.1, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.23.0) - parser (>= 3.1.1.0) - rubocop-config-umbrellio (1.35.0.69) - rubocop (~> 1.35.0) - rubocop-performance (~> 1.14.0) - rubocop-rails (~> 2.15.0) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) + rubocop-capybara (2.21.0) + rubocop (~> 1.41) + rubocop-config-umbrellio (1.63.0.93) + rubocop (~> 1.63.0) + rubocop-performance (~> 1.21.0) + rubocop-rails (~> 2.24.0) rubocop-rake (~> 0.6.0) - rubocop-rspec (~> 2.12.0) + rubocop-rspec (~> 2.29.0) rubocop-sequel (~> 0.3.3) - rubocop-performance (1.14.3) - rubocop (>= 1.7.0, < 2.0) - rubocop-ast (>= 0.4.0) - rubocop-rails (2.15.2) + rubocop-factory_bot (2.26.1) + rubocop (~> 1.61) + rubocop-performance (1.21.1) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails (2.24.1) activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 1.7.0, < 2.0) + rubocop (>= 1.33.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) rubocop-rake (0.6.0) rubocop (~> 1.0) - rubocop-rspec (2.12.1) - rubocop (~> 1.31) + rubocop-rspec (2.29.2) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + rubocop-rspec_rails (~> 2.28) + rubocop-rspec_rails (2.29.1) + rubocop (~> 1.61) rubocop-sequel (0.3.4) rubocop (~> 1.0) - ruby-progressbar (1.11.0) - ruby2_keywords (0.0.5) - semantic_logger (4.11.0) + ruby-progressbar (1.13.0) + semantic_logger (4.15.0) concurrent-ruby (~> 1.0) sentry-raven (3.1.2) faraday (>= 1.0) - sentry-ruby (5.5.0) + sentry-ruby (5.18.0) + bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sequel (5.61.0) - simplecov (0.21.2) + sequel (5.81.0) + bigdecimal + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov-lcov (0.8.0) simplecov_json_formatter (0.1.4) - stringio (3.1.0) + stringio (3.1.1) + strscan (3.1.0) symbiont-ruby (0.7.0) - thor (1.3.0) + thor (1.3.1) timeout (0.4.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - umbrellio-sequel-plugins (0.10.0.86) + umbrellio-sequel-plugins (0.15.0.198) sequel symbiont-ruby - unicode-display_width (2.3.0) - webrick (1.7.0) + unicode-display_width (2.5.0) + uri (0.13.0) + webrick (1.8.1) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - yard (0.9.28) - webrick (~> 1.7.0) - zeitwerk (2.6.12) + yard (0.9.36) + zeitwerk (2.6.16) PLATFORMS ruby @@ -309,4 +328,4 @@ DEPENDENCIES yard BUNDLED WITH - 2.3.24 + 2.5.9 diff --git a/lamian.gemspec b/lamian.gemspec index cd4eff6..35675ac 100644 --- a/lamian.gemspec +++ b/lamian.gemspec @@ -21,18 +21,4 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "rails", ">= 4.2" - - spec.add_development_dependency "bundler-audit" - spec.add_development_dependency "ci-helper" - spec.add_development_dependency "launchy" - spec.add_development_dependency "pry" - spec.add_development_dependency "rake" - spec.add_development_dependency "rspec" - spec.add_development_dependency "rubocop-config-umbrellio" - spec.add_development_dependency "semantic_logger" - spec.add_development_dependency "sentry-raven" - spec.add_development_dependency "sentry-ruby" - spec.add_development_dependency "simplecov" - spec.add_development_dependency "simplecov-lcov" - spec.add_development_dependency "yard" end diff --git a/lib/lamian.rb b/lib/lamian.rb index 36582d7..091fc13 100644 --- a/lib/lamian.rb +++ b/lib/lamian.rb @@ -9,6 +9,7 @@ module Lamian autoload :Config, "lamian/config" autoload :Logger, "lamian/logger" autoload :LoggerExtension, "lamian/logger_extension" + autoload :LogDevice, "lamian/log_device" autoload :Middleware, "lamian/middleware" autoload :RavenContextExtension, "lamian/raven_context_extension" autoload :SidekiqRavenMiddleware, "lamian/sidekiq_raven_middleware" diff --git a/lib/lamian/config.rb b/lib/lamian/config.rb index 5839fc0..695b3e9 100644 --- a/lib/lamian/config.rb +++ b/lib/lamian/config.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true -require "logger" - module Lamian # General lamian configuration class # @attr formatter [Logger::Foramtter] # formatter to use in lamian, global + # @attr max_log_lines [Integer] + # max number of most recent log lines to store, defaults to 5000 # @attr raven_log_size_limit [Integer] # size limit when sending lamian log to sentry, defaults to +500_000+ - Config = Struct.new(:formatter, :raven_log_size_limit) do + Config = Struct.new(:formatter, :max_log_lines, :raven_log_size_limit) do def initialize self.formatter = ::Logger::Formatter.new + self.max_log_lines = 5000 self.raven_log_size_limit = 500_000 end end diff --git a/lib/lamian/log_device.rb b/lib/lamian/log_device.rb new file mode 100644 index 0000000..b3ba958 --- /dev/null +++ b/lib/lamian/log_device.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Lamian + class LogDevice # :nodoc: + def initialize(size = Lamian.config.max_log_lines) + self.size = size + self.lines = [] + end + + def write(string) # :nodoc: + lines << string + lines.shift if lines.size > size + true + end + + def string # :nodoc: + lines.join + end + + private + + attr_accessor :size, :lines + end +end diff --git a/lib/lamian/logger.rb b/lib/lamian/logger.rb index ba8c3c8..3009188 100644 --- a/lib/lamian/logger.rb +++ b/lib/lamian/logger.rb @@ -23,8 +23,7 @@ def initialize # @see Lamian.run # Collects logs sent inside block def run - logdevs.push(StringIO.new) - + logdevs.push(Lamian::LogDevice.new) yield ensure logdevs.pop diff --git a/lib/lamian/version.rb b/lib/lamian/version.rb index 383df06..97b8a97 100644 --- a/lib/lamian/version.rb +++ b/lib/lamian/version.rb @@ -13,5 +13,5 @@ module Lamian # According to this, it is enough to specify '~> a.b' # if private API was not used and to specify '~> a.b.c' if it was - VERSION = "1.8.0" + VERSION = "1.9.0" end diff --git a/spec/lamian/log_device_spec.rb b/spec/lamian/log_device_spec.rb new file mode 100644 index 0000000..467b013 --- /dev/null +++ b/spec/lamian/log_device_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +describe Lamian::LogDevice do + subject(:logdev) { described_class.new(5) } + + it "saves log" do + logdev.write("Hello ") + logdev.write("world!") + expect(logdev.string).to eq("Hello world!") + end + + it "only stores 5 latest log lines" do + 8.times { |x| logdev.write(x + 1) } + expect(logdev.string).to eq("45678") + end +end diff --git a/spec/lamian/semantic_logger_appender_spec.rb b/spec/lamian/semantic_logger_appender_spec.rb index 0811f9a..04e6d52 100644 --- a/spec/lamian/semantic_logger_appender_spec.rb +++ b/spec/lamian/semantic_logger_appender_spec.rb @@ -9,8 +9,7 @@ let(:log) do instance_double(SemanticLogger::Log).tap do |instance| - allow(instance).to receive(:message).and_return("some message") - allow(instance).to receive(:level).and_return(log_level) + allow(instance).to receive_messages(message: "some message", level: log_level) end end let(:log_level) { :debug } diff --git a/spec/support/cool_loggers.rb b/spec/support/cool_loggers.rb index f66e588..51b9851 100644 --- a/spec/support/cool_loggers.rb +++ b/spec/support/cool_loggers.rb @@ -2,7 +2,7 @@ shared_context "cool loggers", :cool_loggers do let(:generic_logger_buffer) { StringIO.new } - let(:generic_logger) { ::Logger.new(generic_logger_buffer) } + let(:generic_logger) { Logger.new(generic_logger_buffer) } let(:cool_formatter) do -> (_severity, _date, _progname, message) { "#{message}\n" }