From 1506b616c10ad963e7a211190904269117a75d88 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Thu, 25 Jul 2024 14:50:19 +0200 Subject: [PATCH] feat(digest): allow modern algorithm --- .github/workflows/rspec.yml | 1 + Appraisals | 4 ++ README.md | 35 +++++++++++------ gemfiles/sidekiq_7.1.gemfile | 2 +- gemfiles/sidekiq_7.2.gemfile | 2 +- gemfiles/sidekiq_7.3.gemfile | 28 ++++++++++++++ lib/sidekiq_unique_jobs/config.rb | 22 +++++++++-- lib/sidekiq_unique_jobs/lock_digest.rb | 7 +++- spec/sidekiq_unique_jobs/lock_digest_spec.rb | 40 +++++++++++++++----- 9 files changed, 115 insertions(+), 26 deletions(-) create mode 100644 gemfiles/sidekiq_7.3.gemfile diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index bea0a85d8..e3a845eab 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -63,6 +63,7 @@ jobs: - sidekiq_7.0 - sidekiq_7.1 - sidekiq_7.2 + - sidekiq_7.3 steps: - uses: actions/checkout@v4 diff --git a/Appraisals b/Appraisals index cc63395ab..34d3512f5 100644 --- a/Appraisals +++ b/Appraisals @@ -15,3 +15,7 @@ end appraise "sidekiq-7.2" do gem "sidekiq", "~> 7.2.0" end + +appraise "sidekiq-7.3" do + gem "sidekiq", "~> 7.3.0" +end diff --git a/README.md b/README.md index ff3976fcd..358a65bd5 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Want to show me some ❤️ for the hard work I do on this gem? You can use the - [sidekiq-global_id](#sidekiq-global_id) - [sidekiq-status](#sidekiq-status) - [Global Configuration](#global-configuration) + - [digest_algorithm](#digest_algorithm) - [debug_lua](#debug_lua) - [lock_timeout](#lock_timeout) - [lock_ttl](#lock_ttl) @@ -734,17 +735,29 @@ Configure SidekiqUniqueJobs in an initializer or the sidekiq initializer on appl ```ruby SidekiqUniqueJobs.configure do |config| - config.logger = Sidekiq.logger # default, change at your own discretion - config.logger_enabled = true # default, disable for test environments - config.debug_lua = false # Turn on when debugging - config.lock_info = false # Turn on when debugging - config.lock_ttl = 600 # Expire locks after 10 minutes - config.lock_timeout = nil # turn off lock timeout - config.max_history = 0 # Turn on when debugging - config.reaper = :ruby # :ruby, :lua or :none/nil - config.reaper_count = 1000 # Stop reaping after this many keys - config.reaper_interval = 600 # Reap orphans every 10 minutes - config.reaper_timeout = 150 # Timeout reaper after 2.5 minutes + config.logger = Sidekiq.logger # default, change at your own discretion + config.logger_enabled = true # default, disable for test environments + config.debug_lua = false # Turn on when debugging + config.lock_info = false # Turn on when debugging + config.lock_ttl = 600 # Expire locks after 10 minutes + config.lock_timeout = nil # turn off lock timeout + config.max_history = 0 # Turn on when debugging + config.reaper = :ruby # :ruby, :lua or :none/nil + config.reaper_count = 1000 # Stop reaping after this many keys + config.reaper_interval = 600 # Reap orphans every 10 minutes + config.reaper_timeout = 150 # Timeout reaper after 2.5 minutes + config.digest_algorithm = :modern # Timeout reaper after 2.5 minutes +end +``` +#### digest_algorithm + +For backwards compatibility this one is set to `:legacy` by the default. If you happen to run into issues with FIPS being enabled on your redis server you might want to set this to `:modern`. + +See: https://github.com/mhenrixon/sidekiq-unique-jobs/issues/848 for explanation + +```ruby +SidekiqUniqueJobs.configure do |config| + config.digest_algorithm = :modern # Timeout reaper after 2.5 minutes end ``` diff --git a/gemfiles/sidekiq_7.1.gemfile b/gemfiles/sidekiq_7.1.gemfile index 140999215..539a3be9a 100644 --- a/gemfiles/sidekiq_7.1.gemfile +++ b/gemfiles/sidekiq_7.1.gemfile @@ -19,7 +19,7 @@ gem "sinatra" gem "timecop" gem "toxiproxy" gem "yard" -gem "sidekiq", "~> 7.0.0" +gem "sidekiq", "~> 7.1.0" platforms :mri do gem "concurrent-ruby-ext" diff --git a/gemfiles/sidekiq_7.2.gemfile b/gemfiles/sidekiq_7.2.gemfile index 140999215..be782364c 100644 --- a/gemfiles/sidekiq_7.2.gemfile +++ b/gemfiles/sidekiq_7.2.gemfile @@ -19,7 +19,7 @@ gem "sinatra" gem "timecop" gem "toxiproxy" gem "yard" -gem "sidekiq", "~> 7.0.0" +gem "sidekiq", "~> 7.2.0" platforms :mri do gem "concurrent-ruby-ext" diff --git a/gemfiles/sidekiq_7.3.gemfile b/gemfiles/sidekiq_7.3.gemfile new file mode 100644 index 000000000..bc1012eb1 --- /dev/null +++ b/gemfiles/sidekiq_7.3.gemfile @@ -0,0 +1,28 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "appraisal" +gem "faraday-retry" +gem "gem-release" +gem "github-markup" +gem "rack-test" +gem "rake", "13.0.3" +gem "reek", ">= 5.3" +gem "rspec" +gem "rspec-benchmark" +gem "rspec-html-matchers" +gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false +gem "sinatra" +gem "timecop" +gem "toxiproxy" +gem "yard" +gem "sidekiq", "~> 7.3.0" + +platforms :mri do + gem "concurrent-ruby-ext" +end + +gemspec path: "../" diff --git a/lib/sidekiq_unique_jobs/config.rb b/lib/sidekiq_unique_jobs/config.rb index 23192823c..a4ac111e0 100644 --- a/lib/sidekiq_unique_jobs/config.rb +++ b/lib/sidekiq_unique_jobs/config.rb @@ -21,7 +21,8 @@ module SidekiqUniqueJobs :reaper_resurrector_enabled, :lock_info, :raise_on_config_error, - :current_redis_version) + :current_redis_version, + :digest_algorithm) # # Shared class for dealing with gem configuration @@ -118,11 +119,9 @@ class Config < ThreadSafeConfig # # @return [3600] check if reaper is dead each 3600 seconds REAPER_RESURRECTOR_INTERVAL = 3600 - # # @return [false] enable reaper resurrector REAPER_RESURRECTOR_ENABLED = false - # # @return [false] while useful it also adds overhead so disable lock_info by default USE_LOCK_INFO = false @@ -132,6 +131,9 @@ class Config < ThreadSafeConfig # # @return [0.0.0] default redis version is only to avoid NoMethodError on nil REDIS_VERSION = "0.0.0" + # + # @return [:legacy] default digest algorithm :modern or :legacy + DIGEST_ALGORITHM = :legacy # # Returns a default configuration @@ -198,6 +200,7 @@ def self.default # rubocop:disable Metrics/MethodLength USE_LOCK_INFO, RAISE_ON_CONFIG_ERROR, REDIS_VERSION, + DIGEST_ALGORITHM, ) end @@ -304,6 +307,19 @@ def add_strategy(name, klass) self.strategies = new_strategies end + # + # Sets digest_algorithm to either :modern or :legacy + # + # @param [Symbol] value + # + # @return [Symbol] the new value + # + def digest_algorithm=(value) + raise ArgumentError, "Invalid digest algorithm: #{value} (should be :modern or :legacy)" unless %i[modern legacy].include?(value) + + super(value) + end + # # The current version of redis # diff --git a/lib/sidekiq_unique_jobs/lock_digest.rb b/lib/sidekiq_unique_jobs/lock_digest.rb index c81ec76ea..b8fb2a31e 100644 --- a/lib/sidekiq_unique_jobs/lock_digest.rb +++ b/lib/sidekiq_unique_jobs/lock_digest.rb @@ -51,7 +51,12 @@ def lock_digest # Creates a namespaced unique digest based on the {#digestable_hash} and the {#lock_prefix} # @return [String] a unique digest def create_digest - digest = OpenSSL::Digest::MD5.hexdigest(dump_json(digestable_hash.sort)) + if (SidekiqUniqueJobs.config.digest_algorithm == :legacy) + digest = OpenSSL::Digest::MD5.hexdigest(dump_json(digestable_hash.sort)) + else + digest = OpenSSL::Digest.new("SHA3-256", dump_json(digestable_hash.sort)).hexdigest + end + "#{lock_prefix}:#{digest}" end diff --git a/spec/sidekiq_unique_jobs/lock_digest_spec.rb b/spec/sidekiq_unique_jobs/lock_digest_spec.rb index 1187874d6..2e50cf8c5 100644 --- a/spec/sidekiq_unique_jobs/lock_digest_spec.rb +++ b/spec/sidekiq_unique_jobs/lock_digest_spec.rb @@ -35,7 +35,7 @@ let(:digest_two) { described_class.new(another_item) } context "with the same unique args" do - let(:another_item) { item } + let(:another_item) { item.dup } it "equals to lock_digest for that item" do expect(lock_digest).to eq(digest_two.lock_digest) @@ -52,18 +52,40 @@ end end - context "when digest is a proc" do - let(:job_class) { MyUniqueJobWithFilterProc } - let(:args) { [1, 2, { "type" => "it" }] } + context "when digest_algorithm is :legacy" do + context "when digest is a proc" do + let(:job_class) { MyUniqueJobWithFilterProc } + let(:args) { [1, 2, { "type" => "it" }] } - it_behaves_like "unique digest" + it_behaves_like "unique digest" + end + + context "when unique_args is a symbol" do + let(:job_class) { MyUniqueJobWithFilterMethod } + let(:args) { [1, 2, { "type" => "it" }] } + + it_behaves_like "unique digest" + end end - context "when unique_args is a symbol" do - let(:job_class) { MyUniqueJobWithFilterMethod } - let(:args) { [1, 2, { "type" => "it" }] } + context "when digest_algorithm is :modern" do + around do |example| + SidekiqUniqueJobs.use_config(digest_algorithm: :modern, &example) + end + + context "when digest is a proc" do + let(:job_class) { MyUniqueJobWithFilterProc } + let(:args) { [1, 2, { "type" => "it" }] } + + it_behaves_like "unique digest" + end - it_behaves_like "unique digest" + context "when unique_args is a symbol" do + let(:job_class) { MyUniqueJobWithFilterMethod } + let(:args) { [1, 2, { "type" => "it" }] } + + it_behaves_like "unique digest" + end end end