Skip to content

Commit

Permalink
feat(digest): allow modern algorithm
Browse files Browse the repository at this point in the history
  • Loading branch information
mhenrixon committed Jul 25, 2024
1 parent bdca185 commit c5139e5
Show file tree
Hide file tree
Showing 22 changed files with 142 additions and 44 deletions.
1 change: 1 addition & 0 deletions .github/workflows/rspec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ jobs:
- sidekiq_7.0
- sidekiq_7.1
- sidekiq_7.2
- sidekiq_7.3

steps:
- uses: actions/checkout@v4
Expand Down
8 changes: 7 additions & 1 deletion .rubocop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,13 @@ RSpec/ExpectActual:
RSpec/ExpectChange:
EnforcedStyle: block

RSpec/FilePath:
RSpec/SpecFilePathFormat:
Enabled: true
Exclude:
- spec/performance/locksmith_spec.rb
- spec/performance/lock_digest_spec.rb

RSpec/SpecFilePathSuffix:
Enabled: true
Exclude:
- spec/performance/locksmith_spec.rb
Expand Down
4 changes: 4 additions & 0 deletions Appraisals
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@ end
appraise "sidekiq-7.2" do
gem "sidekiq", "~> 7.2.0"
end

appraise "sidekiq-7.3" do
gem "sidekiq", "~> 7.3.0"
end
35 changes: 24 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ Want to show me some ❤️ for the hard work I do on this gem? You can use the
- [sidekiq-global_id](#sidekiq-global_id)
- [sidekiq-status](#sidekiq-status)
- [Global Configuration](#global-configuration)
- [digest_algorithm](#digest_algorithm)
- [debug_lua](#debug_lua)
- [lock_timeout](#lock_timeout)
- [lock_ttl](#lock_ttl)
Expand Down Expand Up @@ -734,17 +735,29 @@ Configure SidekiqUniqueJobs in an initializer or the sidekiq initializer on appl

```ruby
SidekiqUniqueJobs.configure do |config|
config.logger = Sidekiq.logger # default, change at your own discretion
config.logger_enabled = true # default, disable for test environments
config.debug_lua = false # Turn on when debugging
config.lock_info = false # Turn on when debugging
config.lock_ttl = 600 # Expire locks after 10 minutes
config.lock_timeout = nil # turn off lock timeout
config.max_history = 0 # Turn on when debugging
config.reaper = :ruby # :ruby, :lua or :none/nil
config.reaper_count = 1000 # Stop reaping after this many keys
config.reaper_interval = 600 # Reap orphans every 10 minutes
config.reaper_timeout = 150 # Timeout reaper after 2.5 minutes
config.logger = Sidekiq.logger # default, change at your own discretion
config.logger_enabled = true # default, disable for test environments
config.debug_lua = false # Turn on when debugging
config.lock_info = false # Turn on when debugging
config.lock_ttl = 600 # Expire locks after 10 minutes
config.lock_timeout = nil # turn off lock timeout
config.max_history = 0 # Turn on when debugging
config.reaper = :ruby # :ruby, :lua or :none/nil
config.reaper_count = 1000 # Stop reaping after this many keys
config.reaper_interval = 600 # Reap orphans every 10 minutes
config.reaper_timeout = 150 # Timeout reaper after 2.5 minutes
config.digest_algorithm = :modern # Timeout reaper after 2.5 minutes
end
```
#### digest_algorithm

For backwards compatibility this one is set to `:legacy` by the default. If you happen to run into issues with FIPS being enabled on your redis server you might want to set this to `:modern`.

See: https://github.com/mhenrixon/sidekiq-unique-jobs/issues/848 for explanation

```ruby
SidekiqUniqueJobs.configure do |config|
config.digest_algorithm = :modern # Timeout reaper after 2.5 minutes
end
```

Expand Down
2 changes: 1 addition & 1 deletion gemfiles/sidekiq_7.1.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ gem "sinatra"
gem "timecop"
gem "toxiproxy"
gem "yard"
gem "sidekiq", "~> 7.0.0"
gem "sidekiq", "~> 7.1.0"

platforms :mri do
gem "concurrent-ruby-ext"
Expand Down
2 changes: 1 addition & 1 deletion gemfiles/sidekiq_7.2.gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ gem "sinatra"
gem "timecop"
gem "toxiproxy"
gem "yard"
gem "sidekiq", "~> 7.0.0"
gem "sidekiq", "~> 7.2.0"

platforms :mri do
gem "concurrent-ruby-ext"
Expand Down
28 changes: 28 additions & 0 deletions gemfiles/sidekiq_7.3.gemfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# This file was generated by Appraisal

source "https://rubygems.org"

gem "appraisal"
gem "faraday-retry"
gem "gem-release"
gem "github-markup"
gem "rack-test"
gem "rake", "13.0.3"
gem "reek", ">= 5.3"
gem "rspec"
gem "rspec-benchmark"
gem "rspec-html-matchers"
gem "rspec-its"
gem "rubocop-mhenrixon"
gem "simplecov-sublime", ">= 0.21.2", require: false
gem "sinatra"
gem "timecop"
gem "toxiproxy"
gem "yard"
gem "sidekiq", "~> 7.3.0"

platforms :mri do
gem "concurrent-ruby-ext"
end

gemspec path: "../"
24 changes: 21 additions & 3 deletions lib/sidekiq_unique_jobs/config.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ module SidekiqUniqueJobs
:reaper_resurrector_enabled,
:lock_info,
:raise_on_config_error,
:current_redis_version)
:current_redis_version,
:digest_algorithm)

#
# Shared class for dealing with gem configuration
Expand Down Expand Up @@ -118,11 +119,9 @@ class Config < ThreadSafeConfig
#
# @return [3600] check if reaper is dead each 3600 seconds
REAPER_RESURRECTOR_INTERVAL = 3600

#
# @return [false] enable reaper resurrector
REAPER_RESURRECTOR_ENABLED = false

#
# @return [false] while useful it also adds overhead so disable lock_info by default
USE_LOCK_INFO = false
Expand All @@ -132,6 +131,9 @@ class Config < ThreadSafeConfig
#
# @return [0.0.0] default redis version is only to avoid NoMethodError on nil
REDIS_VERSION = "0.0.0"
#
# @return [:legacy] default digest algorithm :modern or :legacy
DIGEST_ALGORITHM = :legacy

#
# Returns a default configuration
Expand Down Expand Up @@ -198,6 +200,7 @@ def self.default # rubocop:disable Metrics/MethodLength
USE_LOCK_INFO,
RAISE_ON_CONFIG_ERROR,
REDIS_VERSION,
DIGEST_ALGORITHM,
)
end

Expand Down Expand Up @@ -304,6 +307,21 @@ def add_strategy(name, klass)
self.strategies = new_strategies
end

#
# Sets digest_algorithm to either :modern or :legacy
#
# @param [Symbol] value
#
# @return [Symbol] the new value
#
def digest_algorithm=(value)
unless [:modern, :legacy].include?(value)
raise ArgumentError, "Invalid digest algorithm: #{value} (should be :modern or :legacy)"
end

super
end

#
# The current version of redis
#
Expand Down
2 changes: 1 addition & 1 deletion lib/sidekiq_unique_jobs/digests.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class Digests < Redis::SortedSet
EMPTY_KEYS_SEGMENT = ["", "", "", ""].freeze

def initialize(digests_key = DIGESTS)
super(digests_key)
super
end

#
Expand Down
2 changes: 1 addition & 1 deletion lib/sidekiq_unique_jobs/lock/while_executing.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class WhileExecuting < BaseLock
# @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
#
def initialize(item, callback, redis_pool = nil)
super(item, callback, redis_pool)
super
append_unique_key_suffix
end

Expand Down
7 changes: 6 additions & 1 deletion lib/sidekiq_unique_jobs/lock_digest.rb
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,12 @@ def lock_digest
# Creates a namespaced unique digest based on the {#digestable_hash} and the {#lock_prefix}
# @return [String] a unique digest
def create_digest
digest = OpenSSL::Digest::MD5.hexdigest(dump_json(digestable_hash.sort))
digest = if SidekiqUniqueJobs.config.digest_algorithm == :legacy
OpenSSL::Digest::MD5.hexdigest(dump_json(digestable_hash.sort))
else
OpenSSL::Digest.new("SHA3-256", dump_json(digestable_hash.sort)).hexdigest
end

"#{lock_prefix}:#{digest}"
end

Expand Down
2 changes: 1 addition & 1 deletion lib/sidekiq_unique_jobs/on_conflict/replace.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class Replace < OnConflict::Strategy
# @param [Hash] item sidekiq job hash
#
def initialize(item, redis_pool = nil)
super(item, redis_pool)
super
@queue = item[QUEUE]
@lock_digest = item[LOCK_DIGEST]
end
Expand Down
2 changes: 1 addition & 1 deletion lib/sidekiq_unique_jobs/on_conflict/reschedule.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class Reschedule < OnConflict::Strategy

# @param [Hash] item sidekiq job hash
def initialize(item, redis_pool = nil)
super(item, redis_pool)
super
self.job_class = item[CLASS]
end

Expand Down
3 changes: 2 additions & 1 deletion lib/sidekiq_unique_jobs/orphans/manager.rb
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ module Manager
#
# @return [SidekiqUniqueJobs::TimerTask] the task that was started
#
def start(test_task = nil) # rubocop:disable
# rubocop:disable
def start(test_task = nil)
return if disabled?
return if registered?

Expand Down
2 changes: 1 addition & 1 deletion lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ class RubyReaper < Reaper
# @param [Redis] conn a connection to redis
#
def initialize(conn)
super(conn)
super
@digests = SidekiqUniqueJobs::Digests.new
@scheduled = Redis::SortedSet.new(SCHEDULE)
@retried = Redis::SortedSet.new(RETRY)
Expand Down
4 changes: 2 additions & 2 deletions lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ module UniqueExtension
#
def delete(score, job_id)
entry = find_job(job_id)
SidekiqUniqueJobs::Unlockable.delete!(entry.item) if super(score, job_id)
SidekiqUniqueJobs::Unlockable.delete!(entry.item) if super
entry
end
end
Expand Down Expand Up @@ -132,7 +132,7 @@ def clear
# @param [String] value a sidekiq job hash
#
def delete_by_value(name, value)
SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(value)) if super(name, value)
SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(value)) if super
end
end

Expand Down
2 changes: 1 addition & 1 deletion lib/sidekiq_unique_jobs/testing.rb
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ module Overrides
def sidekiq_options(options = {})
SidekiqUniqueJobs.validate_worker!(options) if SidekiqUniqueJobs.config.raise_on_config_error

super(options)
super
end

#
Expand Down
4 changes: 2 additions & 2 deletions spec/performance/lock_digest_spec.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# frozen_string_literal: true

# rubocop:disable RSpec/SpecFilePathFormat, RSpec/FilePath
# rubocop:disable RSpec/SpecFilePathFormat
RSpec.describe SidekiqUniqueJobs::LockDigest, :perf do
let(:lock_digest) { described_class.new(item) }
let(:job_class) { UntilExecutedJob }
Expand Down Expand Up @@ -51,4 +51,4 @@
end
end
end
# rubocop:enable RSpec/SpecFilePathFormat, RSpec/FilePath
# rubocop:enable RSpec/SpecFilePathFormat
4 changes: 2 additions & 2 deletions spec/performance/locksmith_spec.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# frozen_string_literal: true

# rubocop:disable RSpec/SpecFilePathFormat, RSpec/FilePath
# rubocop:disable RSpec/SpecFilePathFormat
RSpec.describe SidekiqUniqueJobs::Locksmith, :perf do
let(:locksmith_one) { described_class.new(item_one) }
let(:locksmith_two) { described_class.new(item_two) }
Expand Down Expand Up @@ -46,4 +46,4 @@
expect { locksmith_one.unlock }.to perform_under(1).ms
end
end
# rubocop:enable RSpec/SpecFilePathFormat, RSpec/FilePath
# rubocop:enable RSpec/SpecFilePathFormat
4 changes: 2 additions & 2 deletions spec/sidekiq_unique_jobs/configuration_spec.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# frozen_string_literal: true

# rubocop:disable RSpec/SpecFilePathFormat, RSpec/FilePath
# rubocop:disable RSpec/SpecFilePathFormat
RSpec.describe SidekiqUniqueJobs do
describe "define custom lock strategies" do
subject(:middleware_call) do
Expand Down Expand Up @@ -65,4 +65,4 @@ def lock
end
end
end
# rubocop:enable RSpec/SpecFilePathFormat, RSpec/FilePath
# rubocop:enable RSpec/SpecFilePathFormat
40 changes: 31 additions & 9 deletions spec/sidekiq_unique_jobs/lock_digest_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
let(:digest_two) { described_class.new(another_item) }

context "with the same unique args" do
let(:another_item) { item }
let(:another_item) { item.dup }

it "equals to lock_digest for that item" do
expect(lock_digest).to eq(digest_two.lock_digest)
Expand All @@ -52,18 +52,40 @@
end
end

context "when digest is a proc" do
let(:job_class) { MyUniqueJobWithFilterProc }
let(:args) { [1, 2, { "type" => "it" }] }
context "when digest_algorithm is :legacy" do
context "when digest is a proc" do
let(:job_class) { MyUniqueJobWithFilterProc }
let(:args) { [1, 2, { "type" => "it" }] }

it_behaves_like "unique digest"
it_behaves_like "unique digest"
end

context "when unique_args is a symbol" do
let(:job_class) { MyUniqueJobWithFilterMethod }
let(:args) { [1, 2, { "type" => "it" }] }

it_behaves_like "unique digest"
end
end

context "when unique_args is a symbol" do
let(:job_class) { MyUniqueJobWithFilterMethod }
let(:args) { [1, 2, { "type" => "it" }] }
context "when digest_algorithm is :modern" do
around do |example|
SidekiqUniqueJobs.use_config(digest_algorithm: :modern, &example)
end

context "when digest is a proc" do
let(:job_class) { MyUniqueJobWithFilterProc }
let(:args) { [1, 2, { "type" => "it" }] }

it_behaves_like "unique digest"
end

it_behaves_like "unique digest"
context "when unique_args is a symbol" do
let(:job_class) { MyUniqueJobWithFilterMethod }
let(:args) { [1, 2, { "type" => "it" }] }

it_behaves_like "unique digest"
end
end
end

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# frozen_string_literal: true

# rubocop:disable RSpec/SpecFilePathFormat, RSpec/FilePath, RSpec/DescribeMethod
# rubocop:disable RSpec/SpecFilePathFormat, RSpec/DescribeMethod
RSpec.describe SidekiqUniqueJobs::Middleware::Server, "lock: :until_and_while_executing" do
let(:server) { described_class.new }

Expand Down Expand Up @@ -46,4 +46,4 @@
end
end
end
# rubocop:enable RSpec/SpecFilePathFormat, RSpec/FilePath, RSpec/DescribeMethod
# rubocop:enable RSpec/SpecFilePathFormat, RSpec/DescribeMethod

0 comments on commit c5139e5

Please sign in to comment.