From 9ae28fa142ec0a76f8e19432fa3947664dd46b45 Mon Sep 17 00:00:00 2001 From: Masahiro Date: Thu, 16 Jan 2025 11:27:07 +0900 Subject: [PATCH 1/3] Uninstall lru_redux and install sin_lru_redux --- logstash-filter-memoize.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-filter-memoize.gemspec b/logstash-filter-memoize.gemspec index e57707e..6f6690d 100644 --- a/logstash-filter-memoize.gemspec +++ b/logstash-filter-memoize.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" - s.add_runtime_dependency 'lru_redux' + s.add_runtime_dependency 'sin_lru_redux' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-filter-ruby' s.add_development_dependency 'logstash-filter-sleep' From 21c22a0b0a00d80908a2935bf3edf1323226c524 Mon Sep 17 00:00:00 2001 From: Masahiro Date: Thu, 16 Jan 2025 11:55:38 +0900 Subject: [PATCH 2/3] Set ignore_nil to true for saving memory --- lib/logstash/filters/memoize.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/filters/memoize.rb b/lib/logstash/filters/memoize.rb index 7e6075c..fc4948c 100644 --- a/lib/logstash/filters/memoize.rb +++ b/lib/logstash/filters/memoize.rb @@ -55,7 +55,7 @@ class LogStash::Filters::Memoize < LogStash::Filters::Base def register @filter = LogStash::Plugin.lookup("filter", @filter_name).new(@filter_options) @filter.register - @cache = ::LruRedux::TTL::ThreadSafeCache.new(@cache_size, @ttl) + @cache = ::LruRedux::TTL::ThreadSafeCache.new(@cache_size, @ttl, true) end # def register public From e6ebc24bcd2398e2687f462890f940d7376118f4 Mon Sep 17 00:00:00 2001 From: Masahiro Date: Thu, 16 Jan 2025 11:27:37 +0900 Subject: [PATCH 3/3] Refactor --- Rakefile | 2 +- lib/logstash/filters/memoize.rb | 44 ++++++------ logstash-filter-memoize.gemspec | 8 +-- spec/filters/memoize_spec.rb | 117 ++++++++++++++++---------------- spec/spec_helper.rb | 3 +- 5 files changed, 85 insertions(+), 89 deletions(-) diff --git a/Rakefile b/Rakefile index d50e796..e0039b3 100644 --- a/Rakefile +++ b/Rakefile @@ -1 +1 @@ -require "logstash/devutils/rake" +require 'logstash/devutils/rake' diff --git a/lib/logstash/filters/memoize.rb b/lib/logstash/filters/memoize.rb index fc4948c..4450799 100644 --- a/lib/logstash/filters/memoize.rb +++ b/lib/logstash/filters/memoize.rb @@ -1,7 +1,6 @@ -# encoding: utf-8 -require "logstash/filters/base" -require "logstash/namespace" -require "lru_redux" +require 'logstash/filters/base' +require 'logstash/namespace' +require 'lru_redux' # This filter provides https://en.wikipedia.org/wiki/Memoization[memoization] to wrapped filter. # Internally, It based on https://en.wikipedia.org/wiki/Cache_replacement_policies#LRU[LRU] cache algorithm. @@ -12,26 +11,25 @@ # filter { # memoize { # key => "%{host}" <1> -# fields => ["host_owner", "host_location"] <2> -# filter_name => "elasticsearch" <3> +# fields => ['host_owner', 'host_location'] <2> +# filter_name => 'elasticsearch' <3> # filter_options => { <4> # query => "host:%{host}" -# index => "known_host" +# index => 'known_host' # fields => { -# "host_owner" => "host_owner" -# "host_location" => "host_location" +# 'host_owner' => 'host_owner' +# 'host_location' => 'host_location' # } # } # } # } # -------------------------------------------------- -# +# # * When an event with a new <1> key comes in, execute wrapped <3> <4> filter and caches the <2> fields value. # * When an event with a same <1> key comes in, sets cached value to target <2> fields without wrapped <3> <4> filter execution. # class LogStash::Filters::Memoize < LogStash::Filters::Base - - config_name "memoize" + config_name 'memoize' # The key to use caching and retrieving values. It can be dynamic and include parts of the event using the %{field}. config :key, :validate => :string, :required => true @@ -50,30 +48,28 @@ class LogStash::Filters::Memoize < LogStash::Filters::Base # The TTL(Time To Live) in second of cached value. config :ttl, :validate => :number - - public + def register - @filter = LogStash::Plugin.lookup("filter", @filter_name).new(@filter_options) + @filter = LogStash::Plugin.lookup('filter', @filter_name).new(@filter_options) @filter.register @cache = ::LruRedux::TTL::ThreadSafeCache.new(@cache_size, @ttl, true) - end # def register + end - public def filter(event) - formattedKey = event.sprintf(@key); - result = @cache[formattedKey] + formatted_key = event.sprintf(@key) + result = @cache[formatted_key] if !result.nil? - @logger.debug("Cached value found.", :key => formattedKey, :value => result) if @logger.debug? + @logger.debug('Cached value found.', :key => formatted_key, :value => result) if @logger.debug? @fields.each { |field| event.set(field, result[field]) } else - @logger.debug("Cached value not found. Do filter.", :key => formattedKey, :filter => @filter) if @logger.debug? + @logger.debug('Cached value not found. Do filter.', :key => formatted_key, :filter => @filter) if @logger.debug? @filter.filter(event) @fields.each { |field| (result ||= {})[field] = event.get(field) } - @cache[formattedKey] = result + @cache[formatted_key] = result end # filter_matched should go in the last line of our successful code filter_matched(event) - end # def filter -end # class LogStash::Filters::Memoize + end +end diff --git a/logstash-filter-memoize.gemspec b/logstash-filter-memoize.gemspec index 6f6690d..2f215c8 100644 --- a/logstash-filter-memoize.gemspec +++ b/logstash-filter-memoize.gemspec @@ -10,15 +10,15 @@ Gem::Specification.new do |s| s.require_paths = ['lib'] # Files - s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] - # Tests + s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT'] + # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) # Special flag to let us know this is actually a logstash plugin - s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } + s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter' } # Gem dependencies - s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" + s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0' s.add_runtime_dependency 'sin_lru_redux' s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-filter-ruby' diff --git a/spec/filters/memoize_spec.rb b/spec/filters/memoize_spec.rb index 0051ed3..c6d287d 100644 --- a/spec/filters/memoize_spec.rb +++ b/spec/filters/memoize_spec.rb @@ -1,45 +1,45 @@ -# encoding: utf-8 require_relative '../spec_helper' -require "logstash/filters/memoize" +require 'logstash/filters/memoize' -describe LogStash::Filters::Memoize do - describe "basic work" do - let(:config) do <<-CONFIG - filter { - ruby { - code => "event.set('time', Time.now.to_f)" - } +RSpec.describe LogStash::Filters::Memoize do + describe 'basic work' do + let(:config) do + <<-CONFIG + filter { + ruby { + code => "event.set('time', Time.now.to_f)" + } - memoize { - key => "%{key}" - fields => ["result"] - filter_name => "ruby" - filter_options => { - code => "event.set('result', event.get('time'))" + memoize { + key => "%{key}" + fields => ['result'] + filter_name => 'ruby' + filter_options => { + code => "event.set('result', event.get('time'))" + } } } - } - CONFIG + CONFIG end sample([ - {"key" => "cache_key"}, - {"key" => "cache_key2"}, - {"key" => "cache_key"} + { 'key' => 'cache_key' }, + { 'key' => 'cache_key2' }, + { 'key' => 'cache_key' } ]) do # If key is different, result different also. - expect(subject[0].get("time")).to eq(subject[0].get("result")) - expect(subject[1].get("time")).to eq(subject[1].get("result")) - expect(subject[0].get("time")).not_to eq(subject[1].get("result")) - + expect(subject[0].get('time')).to eq(subject[0].get('result')) + expect(subject[1].get('time')).to eq(subject[1].get('result')) + expect(subject[0].get('time')).not_to eq(subject[1].get('result')) + # If key is same, cache works - expect(subject[0].get("time")).to eq(subject[0].get("result")) - expect(subject[2].get("time")).not_to eq(subject[2].get("result")) - expect(subject[0].get("time")).to eq(subject[2].get("result")) + expect(subject[0].get('time')).to eq(subject[0].get('result')) + expect(subject[2].get('time')).not_to eq(subject[2].get('result')) + expect(subject[0].get('time')).to eq(subject[2].get('result')) end end - describe "ttl work" do + describe 'ttl work' do let(:config) do <<-CONFIG filter { ruby { @@ -48,8 +48,8 @@ memoize { key => "%{key}" - fields => ["result"] - filter_name => "ruby" + fields => ['result'] + filter_name => 'ruby' filter_options => { code => "event.set('result', event.get('time'))" } @@ -57,53 +57,54 @@ } sleep { - time => "2" + time => '2' } } CONFIG end sample([ - {"key" => "cache_key"}, - {"key" => "cache_key"} + { 'key' => 'cache_key' }, + { 'key' => 'cache_key' } ]) do # After ttl seconds, cached value are delete - expect(subject[0].get("time")).to eq(subject[0].get("result")) - expect(subject[1].get("time")).to eq(subject[1].get("result")) - expect(subject[0].get("time")).not_to eq(subject[1].get("result")) + expect(subject[0].get('time')).to eq(subject[0].get('result')) + expect(subject[1].get('time')).to eq(subject[1].get('result')) + expect(subject[0].get('time')).not_to eq(subject[1].get('result')) end end - describe "cache_size work" do - let(:config) do <<-CONFIG - filter { - ruby { - code => "event.set('time', Time.now.to_f)" - } + describe 'cache_size work' do + let(:config) do + <<-CONFIG + filter { + ruby { + code => "event.set('time', Time.now.to_f)" + } - memoize { - key => "%{key}" - fields => ["result"] - filter_name => "ruby" - filter_options => { - code => "event.set('result', event.get('time'))" + memoize { + key => "%{key}" + fields => ['result'] + filter_name => 'ruby' + filter_options => { + code => "event.set('result', event.get('time'))" + } + cache_size => 1 } - cache_size => 1 } - } - CONFIG + CONFIG end sample([ - {"key" => "cache_key"}, - {"key" => "cache_key2"}, - {"key" => "cache_key"} + { 'key' => 'cache_key' }, + { 'key' => 'cache_key2' }, + { 'key' => 'cache_key' } ]) do # If cache over cache size, old cached value are delete first - expect(subject[0].get("time")).to eq(subject[0].get("result")) - expect(subject[1].get("time")).to eq(subject[1].get("result")) - expect(subject[2].get("time")).to eq(subject[2].get("result")) - expect(subject[0].get("time")).not_to eq(subject[2].get("result")) + expect(subject[0].get('time')).to eq(subject[0].get('result')) + expect(subject[1].get('time')).to eq(subject[1].get('result')) + expect(subject[2].get('time')).to eq(subject[2].get('result')) + expect(subject[0].get('time')).not_to eq(subject[2].get('result')) end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index dc64aba..1330ae4 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,2 +1 @@ -# encoding: utf-8 -require "logstash/devutils/rspec/spec_helper" +require 'logstash/devutils/rspec/spec_helper'