From f09d41b818286ed5549ac90ccc2ea386be9b5ca4 Mon Sep 17 00:00:00 2001 From: Mikael Henriksson Date: Tue, 21 Sep 2021 17:15:12 +0200 Subject: [PATCH 01/11] Necessary upgrades for Sidekiq v6.2.2 (#639) * Necessary upgrades for Sidekiq v6.2.2 * Adds some coverage for the Sidekiq::Job scenario * Mandatory rubocop commit * Remove a number of gems I never user Also, install gems only when on Mac (prevents additional gems on CI) * Mandatory rubocop commit --- Appraisals | 4 ++ Gemfile | 26 +++----- Guardfile | 61 ------------------- bin/guard | 29 --------- gemfiles/sidekiq_5.0.gemfile | 20 ++---- gemfiles/sidekiq_5.1.gemfile | 20 ++---- gemfiles/sidekiq_5.2.gemfile | 20 ++---- gemfiles/sidekiq_6.0.gemfile | 20 ++---- gemfiles/sidekiq_6.1.gemfile | 20 ++---- gemfiles/sidekiq_6.2.gemfile | 23 +++---- gemfiles/sidekiq_develop.gemfile | 20 ++---- lib/sidekiq_unique_jobs/lock/base_lock.rb | 1 + lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb | 48 +++++++++++---- myapp/Gemfile | 3 +- myapp/spec/rails_helper.rb | 2 +- spec/jobs/another_unique_job_job_spec.rb | 20 ++++++ spec/sidekiq/api_spec.rb | 23 +++++-- .../lock/until_and_while_executing_spec.rb | 27 ++++---- .../have_valid_sidekiq_options_spec.rb | 24 ++++++++ spec/spec_helper.rb | 2 +- spec/support/jobs/another_unique_job_job.rb | 18 ++++++ 21 files changed, 179 insertions(+), 252 deletions(-) delete mode 100644 Guardfile delete mode 100755 bin/guard create mode 100644 spec/jobs/another_unique_job_job_spec.rb create mode 100644 spec/support/jobs/another_unique_job_job.rb diff --git a/Appraisals b/Appraisals index af1b5b92e..fa71bba83 100644 --- a/Appraisals +++ b/Appraisals @@ -23,3 +23,7 @@ end appraise "sidekiq-6.1" do gem "sidekiq", "~> 6.1.0" end + +appraise "sidekiq-6.2" do + gem "sidekiq", "~> 6.2.2" +end diff --git a/Gemfile b/Gemfile index 9fc894ec6..ab1707c34 100644 --- a/Gemfile +++ b/Gemfile @@ -13,39 +13,33 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end if respond_to?(:install_if) install_if -> { RUBY_PLATFORM.include?("darwin") } do + gem "fasterer" gem "fuubar" + gem "github_changelog_generator" gem "pry" + gem "redcarpet", "~> 3.4" gem "rspec-nc" + gem "ruby-prof", ">= 0.17.0", require: false + gem "stackprof", ">= 0.2.9", require: false + gem "test-prof" end end diff --git a/Guardfile b/Guardfile deleted file mode 100644 index 60b86026a..000000000 --- a/Guardfile +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -REEK_ARGS = %w[ - --line-numbers - --color - --documentation - --progress - --single-line - --sort-by smelliness -].freeze - -RUBOCOP_ARGS = %w[ - -P - --format fuubar -].freeze - -RSPEC_OPTIONS = { - cmd: "env COV=true bundle exec rspec", - # cmd_additional_args: "--format documentation", - failed_mode: :focus, - all_on_start: false, -}.freeze - -scope group: :tdd -clearing :on -if /Darwin/.match?(`uname`) - notification :terminal_notifier, app_name: "sidekiq-unique-jobs ::", activate: "com.googlecode.iTerm2" -end - -group :tdd, halt_on_fail: true do - guard :rspec, RSPEC_OPTIONS do - require "guard/rspec/dsl" - dsl = Guard::RSpec::Dsl.new(self) - - # RSpec files - rspec = dsl.rspec - watch(%r{^lib/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" } - watch(%r{^lib/(.+)\.rb$}) { |m| "spec/integration/#{m[1]}_spec.rb" } - watch(%r{^spec/support/workers/(.+)\.rb$}) { |m| "spec/workers/#{m[1]}_spec.rb" } - watch(rspec.spec_helper) { rspec.spec_dir } - watch(rspec.spec_support) { rspec.spec_dir } - watch(rspec.spec_files) - - ruby = dsl.ruby - dsl.watch_spec_files_for(ruby.lib_files) - end - - guard :rubocop, all_on_start: false, cli: RUBOCOP_ARGS do - watch(/.+\.rb$/) - watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) } - end - - guard :reek, all_on_start: false, cli: REEK_ARGS do - watch(/.+\.rb$/) - watch(".reek") - end -end - -guard :bundler do - watch("Gemfile") -end diff --git a/bin/guard b/bin/guard deleted file mode 100755 index 97d05040f..000000000 --- a/bin/guard +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'guard' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -require "pathname" -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", - Pathname.new(__FILE__).realpath) - -bundle_binstub = File.expand_path("bundle", __dir__) - -if File.file?(bundle_binstub) - if /This file was generated by Bundler/.match?(File.read(bundle_binstub, 300)) - load(bundle_binstub) - else - abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. -Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") - end -end - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("guard", "guard") diff --git a/gemfiles/sidekiq_5.0.gemfile b/gemfiles/sidekiq_5.0.gemfile index a933fe67a..924765c67 100644 --- a/gemfiles/sidekiq_5.0.gemfile +++ b/gemfiles/sidekiq_5.0.gemfile @@ -8,9 +8,13 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" @@ -18,23 +22,7 @@ gem "sidekiq", "~> 5.0.0" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end gemspec path: "../" diff --git a/gemfiles/sidekiq_5.1.gemfile b/gemfiles/sidekiq_5.1.gemfile index d237a9b44..3da0c8fa5 100644 --- a/gemfiles/sidekiq_5.1.gemfile +++ b/gemfiles/sidekiq_5.1.gemfile @@ -8,9 +8,13 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" @@ -18,23 +22,7 @@ gem "sidekiq", "~> 5.1.0" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end gemspec path: "../" diff --git a/gemfiles/sidekiq_5.2.gemfile b/gemfiles/sidekiq_5.2.gemfile index 3b35eae83..0ff16e5eb 100644 --- a/gemfiles/sidekiq_5.2.gemfile +++ b/gemfiles/sidekiq_5.2.gemfile @@ -8,9 +8,13 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" @@ -18,23 +22,7 @@ gem "sidekiq", "~> 5.2.0" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end gemspec path: "../" diff --git a/gemfiles/sidekiq_6.0.gemfile b/gemfiles/sidekiq_6.0.gemfile index 0e44b85e4..5d61c41d7 100644 --- a/gemfiles/sidekiq_6.0.gemfile +++ b/gemfiles/sidekiq_6.0.gemfile @@ -8,9 +8,13 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" @@ -18,23 +22,7 @@ gem "sidekiq", "~> 6.0.0" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end gemspec path: "../" diff --git a/gemfiles/sidekiq_6.1.gemfile b/gemfiles/sidekiq_6.1.gemfile index 417f18b12..e4ee36d43 100644 --- a/gemfiles/sidekiq_6.1.gemfile +++ b/gemfiles/sidekiq_6.1.gemfile @@ -8,9 +8,13 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" @@ -18,23 +22,7 @@ gem "sidekiq", "~> 6.1.0" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end gemspec path: "../" diff --git a/gemfiles/sidekiq_6.2.gemfile b/gemfiles/sidekiq_6.2.gemfile index 1ac2498bb..19cb0be4f 100644 --- a/gemfiles/sidekiq_6.2.gemfile +++ b/gemfiles/sidekiq_6.2.gemfile @@ -7,31 +7,22 @@ gem "bundler" gem "gem-release" gem "github-markup" gem "rack-test" -gem "rake" +gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" +gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" -gem "sidekiq", "~> 6.2.0" +gem "sidekiq", "~> 6.2.2" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", require: false - gem "simplecov-sublime", "0.21.0", require: false - gem "travis" end gemspec path: "../" diff --git a/gemfiles/sidekiq_develop.gemfile b/gemfiles/sidekiq_develop.gemfile index 7dcba9f50..cde089c56 100644 --- a/gemfiles/sidekiq_develop.gemfile +++ b/gemfiles/sidekiq_develop.gemfile @@ -8,9 +8,13 @@ gem "gem-release" gem "github-markup" gem "rack-test" gem "rake", "13.0.3" +gem "reek", ">= 5.3" gem "rspec" +gem "rspec-benchmark" gem "rspec-html-matchers" gem "rspec-its" +gem "rubocop-mhenrixon" +gem "simplecov-sublime", ">= 0.21.2", require: false gem "sinatra" gem "timecop" gem "yard" @@ -18,23 +22,7 @@ gem "sidekiq", git: "https://github.com/mperham/sidekiq.git" platforms :mri do gem "concurrent-ruby-ext" - gem "fasterer" - gem "github_changelog_generator" - gem "guard" - gem "guard-bundler" - gem "guard-reek" - gem "guard-rspec" - gem "guard-rubocop" gem "hiredis" - gem "redcarpet", "~> 3.4" - gem "reek", ">= 5.3" - gem "rspec-benchmark" - gem "rubocop-mhenrixon" - gem "ruby-prof", ">= 0.17.0", require: false - gem "simplecov-sublime", ">= 0.21.2", require: false - gem "stackprof", ">= 0.2.9", require: false - gem "test-prof" - gem "travis" end gemspec path: "../" diff --git a/lib/sidekiq_unique_jobs/lock/base_lock.rb b/lib/sidekiq_unique_jobs/lock/base_lock.rb index 1a5441887..e73201d15 100644 --- a/lib/sidekiq_unique_jobs/lock/base_lock.rb +++ b/lib/sidekiq_unique_jobs/lock/base_lock.rb @@ -109,6 +109,7 @@ def prepare_item def lock_failed(origin: :client) reflect(:lock_failed, item) call_strategy(origin: origin) + nil end def call_strategy(origin:) diff --git a/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb b/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb index bdc1dfdb6..f5168c8be 100644 --- a/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +++ b/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb @@ -68,24 +68,46 @@ def delete(score, job_id) prepend UniqueExtension end - # See Sidekiq::Api - class Job - # - # Provides extensions for unlocking jobs that are removed and deleted - # - # @author Mikael Henriksson - # - module UniqueExtension + if Sidekiq.const_defined?("JobRecord") + # See Sidekiq::Api + class JobRecord # - # Wraps the original method to ensure locks for the job are deleted + # Provides extensions for unlocking jobs that are removed and deleted # - def delete - SidekiqUniqueJobs::Unlockable.delete!(item) - super + # @author Mikael Henriksson + # + module UniqueExtension + # + # Wraps the original method to ensure locks for the job are deleted + # + def delete + SidekiqUniqueJobs::Unlockable.delete!(item) + super + end end + + prepend UniqueExtension end + else + # See Sidekiq::Api + class Job + # + # Provides extensions for unlocking jobs that are removed and deleted + # + # @author Mikael Henriksson + # + module UniqueExtension + # + # Wraps the original method to ensure locks for the job are deleted + # + def delete + SidekiqUniqueJobs::Unlockable.delete!(item) + super + end + end - prepend UniqueExtension + prepend UniqueExtension + end end # See Sidekiq::Api diff --git a/myapp/Gemfile b/myapp/Gemfile index 4956517f8..582a40d37 100644 --- a/myapp/Gemfile +++ b/myapp/Gemfile @@ -20,12 +20,11 @@ gem "sinatra" gem "slim-rails" group :development, :test do + gem "debug" gem "dotenv-rails" gem "factory_bot_rails" gem "fuubar" gem "listen" - gem "pry-byebug" - gem "pry-rails" gem "rspec-rails" end diff --git a/myapp/spec/rails_helper.rb b/myapp/spec/rails_helper.rb index 985b40f61..3f6b27c59 100644 --- a/myapp/spec/rails_helper.rb +++ b/myapp/spec/rails_helper.rb @@ -7,7 +7,7 @@ abort("The Rails environment is running in production mode!") if Rails.env.production? require "rspec/rails" -require "pry-byebug" +require "pry" Dir[Rails.root.join("spec", "support", "**", "*.rb")].sort.each { |f| require f } diff --git a/spec/jobs/another_unique_job_job_spec.rb b/spec/jobs/another_unique_job_job_spec.rb new file mode 100644 index 000000000..be7f827f4 --- /dev/null +++ b/spec/jobs/another_unique_job_job_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +if Sidekiq.const_defined?("JobRecord") + RSpec.describe AnotherUniqueJobJob do + it_behaves_like "sidekiq with options" do + let(:options) do + { + "queue" => :working2, + "retry" => 1, + "backtrace" => 10, + "lock" => :until_executed, + } + end + end + + it_behaves_like "a performing worker", splat_arguments: false do + let(:args) { %w[one two] } + end + end +end diff --git a/spec/sidekiq/api_spec.rb b/spec/sidekiq/api_spec.rb index 3075a1901..12eef8c00 100644 --- a/spec/sidekiq/api_spec.rb +++ b/spec/sidekiq/api_spec.rb @@ -49,12 +49,23 @@ end end - describe Sidekiq::Job::UniqueExtension do - it "deletes uniqueness lock on delete" do - jid = JustAWorker.perform_async(roo: "baf") - expect(unique_keys).not_to match_array([]) - Sidekiq::Queue.new("testqueue").find_job(jid).delete - expect(unique_keys).to match_array([]) + if Sidekiq.const_defined?("JobRecord") + describe Sidekiq::JobRecord::UniqueExtension do + it "deletes uniqueness lock on delete" do + jid = JustAWorker.perform_async(roo: "baf") + expect(unique_keys).not_to match_array([]) + Sidekiq::Queue.new("testqueue").find_job(jid).delete + expect(unique_keys).to match_array([]) + end + end + else + describe Sidekiq::Job::UniqueExtension do + it "deletes uniqueness lock on delete" do + jid = JustAWorker.perform_async(roo: "baf") + expect(unique_keys).not_to match_array([]) + Sidekiq::Queue.new("testqueue").find_job(jid).delete + expect(unique_keys).to match_array([]) + end end end diff --git a/spec/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb b/spec/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb index 80ec8089c..df597fa29 100644 --- a/spec/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb +++ b/spec/sidekiq_unique_jobs/lock/until_and_while_executing_spec.rb @@ -1,17 +1,7 @@ # frozen_string_literal: true RSpec.describe SidekiqUniqueJobs::Lock::UntilAndWhileExecuting, redis_db: 3 do - let(:process_one) { described_class.new(item_one, callback) } - let(:runtime_one) { process_one.send(:runtime_lock) } - - let(:process_two) { described_class.new(item_two, callback) } - let(:runtime_two) { process_two.send(:runtime_lock) } - - let(:jid_one) { "jid one" } - let(:jid_two) { "jid two" } - let(:lock_timeout) { nil } - let(:sleepy_time) { 0 } - let(:worker_class) { UntilAndWhileExecutingJob } + let(:process_one) { described_class.new(item_one, callback) } let(:unique) { :until_and_while_executing } let(:queue) { :another_queue } let(:args) { [sleepy_time] } @@ -27,6 +17,21 @@ let(:item_two) do item_one.merge("jid" => jid_two) end + let(:runtime_one) { process_one.send(:runtime_lock) } + + let(:process_two) { described_class.new(item_two, callback) } + let(:runtime_two) { process_two.send(:runtime_lock) } + + let(:jid_one) { "jid one" } + let(:jid_two) { "jid two" } + let(:lock_timeout) { nil } + let(:sleepy_time) { 0 } + + if Sidekiq.const_defined?("JobRecord") + let(:worker_class) { AnotherUniqueJobJob } + else + let(:worker_class) { UntilAndWhileExecutingJob } + end before do allow(runtime_one).to receive(:reflect).and_call_original diff --git a/spec/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options_spec.rb b/spec/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options_spec.rb index 973aee0d8..9583aaba4 100644 --- a/spec/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options_spec.rb +++ b/spec/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options_spec.rb @@ -2,6 +2,30 @@ RSpec.describe SidekiqUniqueJobs::RSpec::Matchers::HaveValidSidekiqOptions do describe "#matches?" do + if Sidekiq.const_defined?("JobRecord") + context "when sidekiq options are valid" do + it { expect(AnotherUniqueJob).to have_valid_sidekiq_options } + end + + context "when sidekiq options lacks `:lock`" do + it "raises SidekiqUniqueJobs::NotUniqueWorker" do + AnotherUniqueJob.use_options({}) do + AnotherUniqueJob.sidekiq_options_hash.delete("lock") + expect { expect(AnotherUniqueJob).to have_valid_sidekiq_options } + .to raise_error(SidekiqUniqueJobs::NotUniqueWorker) + end + end + end + + context "when sidekiq options are invalid" do + it "raises SidekiqUniqueJobs::NotUniqueWorker" do + AnotherUniqueJob.use_options(on_client_conflict: :reject, on_server_conflict: :replace) do + expect(AnotherUniqueJob).not_to have_valid_sidekiq_options + end + end + end + end + context "when sidekiq options are valid" do it { expect(AnotherUniqueJob).to have_valid_sidekiq_options } end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 5012445aa..a50bfa3dd 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -8,7 +8,7 @@ begin require "pry" rescue LoadError - puts "Pry unavailable" + puts "Pry is unavailable" end end diff --git a/spec/support/jobs/another_unique_job_job.rb b/spec/support/jobs/another_unique_job_job.rb new file mode 100644 index 000000000..78541f111 --- /dev/null +++ b/spec/support/jobs/another_unique_job_job.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# :nocov: +if Sidekiq.const_defined?("JobRecord") + require "sidekiq/job" + + class AnotherUniqueJobJob + include Sidekiq::Job + sidekiq_options backtrace: 10, + lock: :until_executed, + queue: :working2, + retry: 1 + + def perform(args) + args + end + end +end From 2ac4c0bcddd783fd0039caefd2a505b845f5788c Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Tue, 21 Sep 2021 20:25:20 +0200 Subject: [PATCH 02/11] Update changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa417d13a..2b7db2b88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [v7.1.6](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.6) (2021-09-21) + +[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.5...v7.1.6) + +**Closed issues:** + +- until\_and\_while\_executing is not running the job at all in sidekiq-unique-jobs 7.1.4 [\#626](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/626) + +**Merged pull requests:** + +- Necessary upgrades for Sidekiq v6.2.2 [\#639](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/639) ([mhenrixon](https://github.com/mhenrixon)) +- Tese to these in README.md [\#633](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/633) ([carrickr](https://github.com/carrickr)) + ## [v7.1.5](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.5) (2021-07-28) [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.4...v7.1.5) From b8d92b11f66c3c4276e1f5d22fe6568e09d85d9a Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Tue, 21 Sep 2021 20:25:21 +0200 Subject: [PATCH 03/11] Bump sidekiq-unique-jobs to 7.1.7 --- lib/sidekiq_unique_jobs/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sidekiq_unique_jobs/version.rb b/lib/sidekiq_unique_jobs/version.rb index c64c1d191..8f6cd2df5 100644 --- a/lib/sidekiq_unique_jobs/version.rb +++ b/lib/sidekiq_unique_jobs/version.rb @@ -3,5 +3,5 @@ module SidekiqUniqueJobs # # @return [String] the current SidekiqUniqueJobs version - VERSION = "7.1.6" + VERSION = "7.1.7" end From 026efd961fda3b7bdc3b1baea43acce4ec5d5235 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Tue, 21 Sep 2021 20:29:09 +0200 Subject: [PATCH 04/11] Update changelog --- .github/workflows/lint.yml | 2 +- .github/workflows/rspec.yml | 2 +- README.md | 3 ++- lib/tasks/changelog.rake | 2 +- update_docs.sh | 4 ++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 471267ffc..288bcb6a3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,7 @@ name: Lint on: pull_request: push: - branches: [ master ] + branches: [ main ] jobs: rubocop: runs-on: ubuntu-latest diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 634f81985..2c1684c19 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -2,7 +2,7 @@ name: RSpec on: pull_request: push: - branches: [ master ] + branches: [ main ] jobs: coverage: services: diff --git a/README.md b/README.md index 2f0f15fe6..6831813f4 100644 --- a/README.md +++ b/README.md @@ -92,10 +92,11 @@ This gem adds unique constraints to sidekiq jobs. The uniqueness is achieved by By default, only one lock for a given hash can be acquired. What happens when a lock can't be acquired is governed by a chosen [Conflict Strategy](#conflict-strategy) strategy. Unless a conflict strategy is chosen -This is the documentation for the master branch. You can find the documentation for each release by navigating to its tag. +This is the documentation for the `main` branch. You can find the documentation for each release by navigating to its tag. Here are links to some of the old versions +- [v7.0.12](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.0.12) - [v6.0.25](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v6.0.25) - [v5.0.10](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v5.0.10) - [v4.0.18](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v4.0.18) diff --git a/lib/tasks/changelog.rake b/lib/tasks/changelog.rake index ff89673fb..8e380e124 100644 --- a/lib/tasks/changelog.rake +++ b/lib/tasks/changelog.rake @@ -16,7 +16,7 @@ task :changelog do COMMIT_CHANGELOG_CMD ||= "git commit -a -m 'Update changelog'" # rubocop:enable Style/MutableConstant - sh("git checkout master") + sh("git checkout main") sh(*CHANGELOG_CMD.push(ENV["CHANGELOG_GITHUB_TOKEN"])) sh(ADD_CHANGELOG_CMD) sh(COMMIT_CHANGELOG_CMD) diff --git a/update_docs.sh b/update_docs.sh index 95c19ec37..eaff3b599 100755 --- a/update_docs.sh +++ b/update_docs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -git checkout master +git checkout main git fetch stash_created=0 @@ -34,4 +34,4 @@ if [[ $stash_created == 1 ]]; then git stash pop fi; -git checkout master +git checkout main From 3c1efa6d1edb8115038cadb2edc3f75365eb9b95 Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Tue, 21 Sep 2021 21:10:23 +0200 Subject: [PATCH 05/11] Update documentation --- lib/sidekiq_unique_jobs/config.rb | 35 +++++++++++++++++++ lib/sidekiq_unique_jobs/deprecation.rb | 30 ++++++++++++++++ lib/sidekiq_unique_jobs/json.rb | 7 ++++ lib/sidekiq_unique_jobs/lock/base_lock.rb | 2 +- lib/sidekiq_unique_jobs/logging.rb | 9 +++++ lib/sidekiq_unique_jobs/orphans/manager.rb | 35 +++++++++++++++++++ .../orphans/ruby_reaper.rb | 2 ++ lib/sidekiq_unique_jobs/reflectable.rb | 13 +++++-- lib/sidekiq_unique_jobs/reflections.rb | 10 ++++++ lib/sidekiq_unique_jobs/server.rb | 12 +++++++ 10 files changed, 152 insertions(+), 3 deletions(-) diff --git a/lib/sidekiq_unique_jobs/config.rb b/lib/sidekiq_unique_jobs/config.rb index 1c53aec3d..23b127491 100644 --- a/lib/sidekiq_unique_jobs/config.rb +++ b/lib/sidekiq_unique_jobs/config.rb @@ -196,26 +196,61 @@ def self.default # rubocop:disable Metrics/MethodLength ) end + # + # Set the default_lock_ttl + # @deprecated + # + # @param [Integer] obj value to set (seconds) + # + # @return [] + # def default_lock_ttl=(obj) warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl=` instead." self.lock_ttl = obj end + # + # Set new value for default_lock_timeout + # @deprecated + # + # @param [Integer] obj value to set (seconds) + # + # @return [Integer] + # def default_lock_timeout=(obj) warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout=` instead." self.lock_timeout = obj end + # + # Default lock TTL (Time To Live) + # @deprecated + # + # @return [nil, Integer] configured value or nil + # def default_lock_ttl warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl` instead." lock_ttl end + # + # Default Lock Timeout + # @deprecated + # + # + # @return [nil, Integer] configured value or nil + # def default_lock_timeout warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout` instead." lock_timeout end + # + # Memoized variable to get the class name + # + # + # @return [String] name of the class + # def class_name @class_name ||= self.class.name end diff --git a/lib/sidekiq_unique_jobs/deprecation.rb b/lib/sidekiq_unique_jobs/deprecation.rb index 08886a00d..74c21471a 100644 --- a/lib/sidekiq_unique_jobs/deprecation.rb +++ b/lib/sidekiq_unique_jobs/deprecation.rb @@ -7,6 +7,13 @@ module SidekiqUniqueJobs # @author Mikael Henriksson # class Deprecation + # + # Mute warnings from this gem in a threaded context + # + # + # @return [void] + # + # @yieldreturn [void] def self.muted orig_val = Thread.current[:uniquejobs_mute_deprecations] Thread.current[:uniquejobs_mute_deprecations] = true @@ -15,21 +22,44 @@ def self.muted Thread.current[:uniquejobs_mute_deprecations] = orig_val end + # + # Check if deprecation warnings have been muted + # + # + # @return [true,false] + # def self.muted? Thread.current[:uniquejobs_mute_deprecations] == true end + # + # Warn about deprecation + # + # @param [String] msg a descriptive reason for why the deprecation + # + # @return [void] + # def self.warn(msg) return if SidekiqUniqueJobs::Deprecation.muted? warn "DEPRECATION WARNING: #{msg}" + nil end + # + # Warn about deprecation and provide a context + # + # @param [String] msg a descriptive reason for why the deprecation + # + # @return [void] + # def self.warn_with_backtrace(msg) return if SidekiqUniqueJobs::Deprecation.muted? trace = "\n\nCALLED FROM:\n#{caller.join("\n")}" warn(msg + trace) + + nil end end end diff --git a/lib/sidekiq_unique_jobs/json.rb b/lib/sidekiq_unique_jobs/json.rb index 2073b1662..40e5f3a36 100644 --- a/lib/sidekiq_unique_jobs/json.rb +++ b/lib/sidekiq_unique_jobs/json.rb @@ -20,6 +20,13 @@ def load_json(string) ::JSON.parse(string) end + # + # Prevents trying JSON.load from raising errors given argument is a hash + # + # @param [String, Hash] string the JSON string to parse + # + # @return [Hash,Array] + # def safe_load_json(string) return string if string.is_a?(Hash) diff --git a/lib/sidekiq_unique_jobs/lock/base_lock.rb b/lib/sidekiq_unique_jobs/lock/base_lock.rb index e73201d15..c78f6a5e6 100644 --- a/lib/sidekiq_unique_jobs/lock/base_lock.rb +++ b/lib/sidekiq_unique_jobs/lock/base_lock.rb @@ -102,7 +102,7 @@ def prepare_item # # Handle when lock failed # - # @param [Symbol] location: :client or :server + # @param [Symbol] origin either `:client` or `:server` # # @return [void] # diff --git a/lib/sidekiq_unique_jobs/logging.rb b/lib/sidekiq_unique_jobs/logging.rb index da26aa184..0e6834821 100644 --- a/lib/sidekiq_unique_jobs/logging.rb +++ b/lib/sidekiq_unique_jobs/logging.rb @@ -92,9 +92,18 @@ def log_error(message_or_exception = nil, item = nil, &block) def log_fatal(message_or_exception = nil, item = nil, &block) message = build_message(message_or_exception, item) logger.fatal(message, &block) + nil end + # + # Build a log message + # + # @param [String, Exception] message_or_exception an entry to log + # @param [Hash] item the sidekiq job hash + # + # @return [String] a complete log entry + # def build_message(message_or_exception, item = nil) return nil if message_or_exception.nil? return message_or_exception if item.nil? diff --git a/lib/sidekiq_unique_jobs/orphans/manager.rb b/lib/sidekiq_unique_jobs/orphans/manager.rb index b296bdccc..9d7e2e4f3 100644 --- a/lib/sidekiq_unique_jobs/orphans/manager.rb +++ b/lib/sidekiq_unique_jobs/orphans/manager.rb @@ -10,10 +10,18 @@ module Orphans module Manager module_function + # + # @return [Float] the amount to add to the reaper interval DRIFT_FACTOR = 0.02 + # + # @return [Symbol] allowed reapers (:ruby or :lua) REAPERS = [:ruby, :lua].freeze + # includes "SidekiqUniqueJobs::Connection" + # @!parse include SidekiqUniqueJobs::Connection include SidekiqUniqueJobs::Connection + # includes "SidekiqUniqueJobs::Logging" + # @!parse include SidekiqUniqueJobs::Logging include SidekiqUniqueJobs::Logging # @@ -65,6 +73,12 @@ def task @task ||= default_task end + # + # A properly configured timer task + # + # + # @return [SidekiqUniqueJobs::TimerTask] + # def default_task SidekiqUniqueJobs::TimerTask.new(timer_task_options) do with_logging_context do @@ -76,6 +90,13 @@ def default_task end end + # + # Store a task to use for scheduled execution + # + # @param [SidekiqUniqueJobs::TimerTask] task the task to use + # + # @return [void] + # def task=(task) @task = task end @@ -201,10 +222,24 @@ def unregister_reaper_process redis { |conn| conn.del(UNIQUE_REAPER) } end + # + # Reaper interval with a little drift + # Redis isn't exact enough so to give a little bufffer, + # we add a tiny value to the reaper interval. + # + # + # @return [Integer] + # def drift_reaper_interval reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i end + # + # Current time (as integer value) + # + # + # @return [Integer] + # def current_timestamp Time.now.to_i end diff --git a/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb b/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb index 775c8d640..87e22b3b6 100644 --- a/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +++ b/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb @@ -10,6 +10,8 @@ module Orphans # @author Mikael Henriksson # class RubyReaper < Reaper + # + # @return [String] the suffix for :RUN locks RUN_SUFFIX = ":RUN" # # @!attribute [r] digests diff --git a/lib/sidekiq_unique_jobs/reflectable.rb b/lib/sidekiq_unique_jobs/reflectable.rb index dc6740481..8c2a1d1ea 100644 --- a/lib/sidekiq_unique_jobs/reflectable.rb +++ b/lib/sidekiq_unique_jobs/reflectable.rb @@ -7,11 +7,20 @@ module SidekiqUniqueJobs # @author Mikael Henriksson # module Reflectable - def reflect(name, *args) - SidekiqUniqueJobs.reflections.dispatch(name, *args) + # + # Reflects on specific event + # + # @param [Symbol] reflection the reflected event + # @param [Array] args arguments to provide to reflector + # + # @return [void] + # + def reflect(reflection, *args) + SidekiqUniqueJobs.reflections.dispatch(reflection, *args) nil rescue UniqueJobsError => ex SidekiqUniqueJobs.logger.error(ex) + nil end end end diff --git a/lib/sidekiq_unique_jobs/reflections.rb b/lib/sidekiq_unique_jobs/reflections.rb index 1ee65091c..162cfd7dd 100644 --- a/lib/sidekiq_unique_jobs/reflections.rb +++ b/lib/sidekiq_unique_jobs/reflections.rb @@ -42,6 +42,14 @@ def initialize @reflections = {} end + # + # Dispatch a reflected event + # + # @param [reflection] reflection the reflected event + # @param [Array] args the arguments to provide to the block + # + # @return [void] + # def dispatch(reflection, *args) if (block = @reflections[reflection]) block.call(*args) @@ -55,6 +63,8 @@ def dispatch(reflection, *args) elsif misconfigured?(reflection) raise NoSuchNotificationError, reflection end + + nil end def configured?(reflection) diff --git a/lib/sidekiq_unique_jobs/server.rb b/lib/sidekiq_unique_jobs/server.rb index 01a99dd58..30bde38a9 100644 --- a/lib/sidekiq_unique_jobs/server.rb +++ b/lib/sidekiq_unique_jobs/server.rb @@ -25,6 +25,12 @@ def self.configure(config) config.death_handlers << death_handler end + # + # Start the sidekiq unique jobs server process + # + # + # @return [void] + # def self.start SidekiqUniqueJobs::UpdateVersion.call SidekiqUniqueJobs::UpgradeLocks.call @@ -32,6 +38,12 @@ def self.start SidekiqUniqueJobs::Orphans::ReaperResurrector.start end + # + # Stop the sidekiq unique jobs server process + # + # + # @return [void] + # def self.stop SidekiqUniqueJobs::Orphans::Manager.stop end From 8f84937986d9a3148c816f47ec220ac7b40a7a3c Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Mon, 2 Aug 2021 17:53:27 +0200 Subject: [PATCH 06/11] Fix `on_conflict` strategy handling This fixes the exception posted below when using `on_conflict: replace` and potentially other bugs related to `on_conflict`. Before [this change][1] `BasicLock.lock` always returned either the acquired lock or the return value of `call_strategy`. Since `Middleware::Client#lock` now only yields when `lock_instance.lock` yields we have to also `yield` inside the lock instances when the lock was acquired via `lock_failed` (i.e. a `on_conflict` strategy). ``` NoMethodError: undefined method `key?' for "f5d69f8fd2e1f3dde8cee02e":String from sidekiq/client.rb:197:in `atomic_push' from sidekiq/client.rb:190:in `block (2 levels) in raw_push' from redis.rb:2489:in `block in multi' from redis.rb:69:in `block in synchronize' from monitor.rb:202:in `synchronize' from monitor.rb:202:in `mon_synchronize' from redis.rb:69:in `synchronize' from redis.rb:2483:in `multi' from sidekiq/client.rb:189:in `block in raw_push' from connection_pool.rb:63:in `block (2 levels) in with' from connection_pool.rb:62:in `handle_interrupt' from connection_pool.rb:62:in `block in with' from connection_pool.rb:59:in `handle_interrupt' from connection_pool.rb:59:in `with' from sidekiq/client.rb:188:in `raw_push' from sidekiq/client.rb:74:in `push' from sidekiq/worker.rb:240:in `client_push' from sidekiq/worker.rb:215:in `perform_in' ``` #590 [1]: s://github.com/mhenrixon/sidekiq-unique-jobs/commit/8c8d54c8b9dea363a7d8b8aeaceb2e82966b8503 --- .../lock/until_and_while_executing.rb | 4 +-- .../lock/until_executed.rb | 4 +-- .../lock/until_executing.rb | 4 +-- lib/sidekiq_unique_jobs/lock/until_expired.rb | 4 +-- .../until_and_while_executing_replace_job.rb | 20 +++++++++++++ ...il_and_while_executing_replace_job_spec.rb | 30 +++++++++++++++++++ 6 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 spec/support/workers/until_and_while_executing_replace_job.rb create mode 100644 spec/workers/until_and_while_executing_replace_job_spec.rb diff --git a/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb b/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb index ced00745a..33ec3e968 100644 --- a/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +++ b/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb @@ -23,8 +23,8 @@ class UntilAndWhileExecuting < BaseLock # @yield to the caller when given a block # def lock(origin: :client) - return lock_failed(origin: origin) unless (token = locksmith.lock) - return yield token if block_given? + token = locksmith.lock || lock_failed(origin: origin) + return yield token if token && block_given? token end diff --git a/lib/sidekiq_unique_jobs/lock/until_executed.rb b/lib/sidekiq_unique_jobs/lock/until_executed.rb index 01a8ec616..82148344b 100644 --- a/lib/sidekiq_unique_jobs/lock/until_executed.rb +++ b/lib/sidekiq_unique_jobs/lock/until_executed.rb @@ -18,8 +18,8 @@ class UntilExecuted < BaseLock # @yield to the caller when given a block # def lock - return lock_failed(origin: :client) unless (token = locksmith.lock) - return yield token if block_given? + token = locksmith.lock || lock_failed(origin: :client) + return yield token if token && block_given? token end diff --git a/lib/sidekiq_unique_jobs/lock/until_executing.rb b/lib/sidekiq_unique_jobs/lock/until_executing.rb index 745136ea9..4b1fa69ce 100644 --- a/lib/sidekiq_unique_jobs/lock/until_executing.rb +++ b/lib/sidekiq_unique_jobs/lock/until_executing.rb @@ -16,8 +16,8 @@ class UntilExecuting < BaseLock # @return [String, nil] the locked jid when properly locked, else nil. # def lock - return lock_failed unless (job_id = locksmith.lock) - return yield job_id if block_given? + job_id = locksmith.lock || lock_failed + return yield job_id if job_id && block_given? job_id end diff --git a/lib/sidekiq_unique_jobs/lock/until_expired.rb b/lib/sidekiq_unique_jobs/lock/until_expired.rb index f498dfab2..f247a966d 100644 --- a/lib/sidekiq_unique_jobs/lock/until_expired.rb +++ b/lib/sidekiq_unique_jobs/lock/until_expired.rb @@ -18,8 +18,8 @@ class UntilExpired < UntilExecuted # @yield to the caller when given a block # def lock - return lock_failed unless (job_id = locksmith.lock) - return yield job_id if block_given? + job_id = locksmith.lock || lock_failed + return yield job_id if job_id && block_given? job_id end diff --git a/spec/support/workers/until_and_while_executing_replace_job.rb b/spec/support/workers/until_and_while_executing_replace_job.rb new file mode 100644 index 000000000..c4d84c891 --- /dev/null +++ b/spec/support/workers/until_and_while_executing_replace_job.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# :nocov: + +class UntilAndWhileExecutingReplaceJob + include Sidekiq::Worker + + sidekiq_options lock: :until_and_while_executing, + queue: :working, + on_conflict: { + client: :replace, + server: :reschedule, + } + + def self.lock_args(args) + [args[0]] + end + + def perform(key); end +end diff --git a/spec/workers/until_and_while_executing_replace_job_spec.rb b/spec/workers/until_and_while_executing_replace_job_spec.rb new file mode 100644 index 000000000..5d90f738c --- /dev/null +++ b/spec/workers/until_and_while_executing_replace_job_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +RSpec.describe UntilAndWhileExecutingReplaceJob do + it_behaves_like "sidekiq with options" do + let(:options) do + { + "queue" => :working, + "retry" => true, + "lock" => :until_and_while_executing, + "on_conflict" => { client: :replace, server: :reschedule }, + } + end + end + + it "replaces the previous job successfully" do + Sidekiq::Testing.disable! do + set = Sidekiq::ScheduledSet.new + + described_class.perform_at(Time.now + 30, "unique", "first argument") + expect(set.size).to eq(1) + expect(set.first.item["args"]).to eq(["unique", "first argument"]) + + described_class.perform_at(Time.now + 30, "unique", "new argument") + expect(set.size).to eq(1) + expect(set.first.item["args"]).to eq(["unique", "new argument"]) + + set.each(&:delete) + end + end +end From 3c08b5b87088579c7bc9dfda27396f16e8e82e8e Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Wed, 1 Sep 2021 18:16:01 +0200 Subject: [PATCH 07/11] Update return types for ``OnConflict#call` methods --- lib/sidekiq_unique_jobs/lock/base_lock.rb | 2 +- lib/sidekiq_unique_jobs/on_conflict/log.rb | 3 ++- lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb | 1 + lib/sidekiq_unique_jobs/on_conflict/reject.rb | 3 +++ lib/sidekiq_unique_jobs/on_conflict/replace.rb | 3 +-- lib/sidekiq_unique_jobs/on_conflict/reschedule.rb | 3 +++ 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/sidekiq_unique_jobs/lock/base_lock.rb b/lib/sidekiq_unique_jobs/lock/base_lock.rb index c78f6a5e6..ef4877693 100644 --- a/lib/sidekiq_unique_jobs/lock/base_lock.rb +++ b/lib/sidekiq_unique_jobs/lock/base_lock.rb @@ -104,7 +104,7 @@ def prepare_item # # @param [Symbol] origin either `:client` or `:server` # - # @return [void] + # @return [String, nil] # def lock_failed(origin: :client) reflect(:lock_failed, item) diff --git a/lib/sidekiq_unique_jobs/on_conflict/log.rb b/lib/sidekiq_unique_jobs/on_conflict/log.rb index a045529ae..6df2778dc 100644 --- a/lib/sidekiq_unique_jobs/on_conflict/log.rb +++ b/lib/sidekiq_unique_jobs/on_conflict/log.rb @@ -12,12 +12,13 @@ class Log < OnConflict::Strategy # Logs an informational message about that the job was not unique # # - # @return [void] + # @return [nil] # def call log_info(<<~MESSAGE.chomp) Skipping job with id (#{item[JID]}) because lock_digest: (#{item[LOCK_DIGEST]}) already exists MESSAGE + nil end end end diff --git a/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb b/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb index 083556138..9ef3e319c 100644 --- a/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +++ b/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb @@ -7,6 +7,7 @@ module OnConflict # @author Mikael Henriksson class NullStrategy < OnConflict::Strategy # Do nothing on conflict + # # @return [nil] def call # NOOP diff --git a/lib/sidekiq_unique_jobs/on_conflict/reject.rb b/lib/sidekiq_unique_jobs/on_conflict/reject.rb index 6591ffcf0..a7c7e2a80 100644 --- a/lib/sidekiq_unique_jobs/on_conflict/reject.rb +++ b/lib/sidekiq_unique_jobs/on_conflict/reject.rb @@ -9,6 +9,8 @@ class Reject < OnConflict::Strategy include SidekiqUniqueJobs::Timing # Send jobs to dead queue + # + # @return [nil] def call log_info { "Adding dead #{item[CLASS]} job #{item[JID]}" } @@ -17,6 +19,7 @@ def call else push_to_deadset end + nil end # diff --git a/lib/sidekiq_unique_jobs/on_conflict/replace.rb b/lib/sidekiq_unique_jobs/on_conflict/replace.rb index d434da9e7..92d38490f 100644 --- a/lib/sidekiq_unique_jobs/on_conflict/replace.rb +++ b/lib/sidekiq_unique_jobs/on_conflict/replace.rb @@ -29,8 +29,7 @@ def initialize(item, redis_pool = nil) # # Replace the old job in the queue # - # - # @return [void] + # @return [String, nil] # # @yield to retry the lock after deleting the old one # diff --git a/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb b/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb index db24c82cb..bcb453c68 100644 --- a/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +++ b/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb @@ -19,6 +19,8 @@ def initialize(item, redis_pool = nil) # Create a new job from the current one. # This will mess up sidekiq stats because a new job is created + # + # @return [nil] def call if sidekiq_worker_class? if worker_class.perform_in(5, *item[ARGS]) @@ -29,6 +31,7 @@ def call else reflect(:unknown_sidekiq_worker, item) end + nil end end end From 10359c27a4787f92d5d89810a301167e5bdcacc2 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Wed, 15 Sep 2021 11:47:01 +0200 Subject: [PATCH 08/11] Add spec testing on_conflict replace using perform_in --- ...until_and_while_executing_replace_job_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/workers/until_and_while_executing_replace_job_spec.rb b/spec/workers/until_and_while_executing_replace_job_spec.rb index 5d90f738c..8e767acf8 100644 --- a/spec/workers/until_and_while_executing_replace_job_spec.rb +++ b/spec/workers/until_and_while_executing_replace_job_spec.rb @@ -27,4 +27,20 @@ set.each(&:delete) end end + + it "replaces the previous job successfully when using perform_in" do + Sidekiq::Testing.disable! do + set = Sidekiq::ScheduledSet.new + + described_class.perform_in(30, "unique", "first argument") + expect(set.size).to eq(1) + expect(set.first.item["args"]).to eq(["unique", "first argument"]) + + described_class.perform_in(30, "unique", "new argument") + expect(set.size).to eq(1) + expect(set.first.item["args"]).to eq(["unique", "new argument"]) + + set.each(&:delete) + end + end end From 66fb5cc0fd933cc79cda804d3bef24e783cf44b2 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Wed, 15 Sep 2021 11:47:08 +0200 Subject: [PATCH 09/11] Add spec for on_conflict reject --- .../until_and_while_executing_reject_job.rb | 20 ++++++++++ ...til_and_while_executing_reject_job_spec.rb | 40 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 spec/support/workers/until_and_while_executing_reject_job.rb create mode 100644 spec/workers/until_and_while_executing_reject_job_spec.rb diff --git a/spec/support/workers/until_and_while_executing_reject_job.rb b/spec/support/workers/until_and_while_executing_reject_job.rb new file mode 100644 index 000000000..4e9c21332 --- /dev/null +++ b/spec/support/workers/until_and_while_executing_reject_job.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# :nocov: + +class UntilAndWhileExecutingRejectJob + include Sidekiq::Worker + + sidekiq_options lock: :until_and_while_executing, + queue: :working, + on_conflict: { + client: :reject, + server: :reject, + } + + def self.lock_args(args) + [args[0]] + end + + def perform(key); end +end diff --git a/spec/workers/until_and_while_executing_reject_job_spec.rb b/spec/workers/until_and_while_executing_reject_job_spec.rb new file mode 100644 index 000000000..3f9f2b1cc --- /dev/null +++ b/spec/workers/until_and_while_executing_reject_job_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +RSpec.describe UntilAndWhileExecutingRejectJob do + it_behaves_like "sidekiq with options" do + let(:options) do + { + "queue" => :working, + "retry" => true, + "lock" => :until_and_while_executing, + "on_conflict" => { client: :reject, server: :reject }, + } + end + end + + it "rejects the job successfully" do + Sidekiq::Testing.disable! do + set = Sidekiq::ScheduledSet.new + + described_class.perform_at(Time.now + 30, 1) + expect(set.size).to eq(1) + + expect(described_class.perform_at(Time.now + 30, 1)).to be_nil + + set.each(&:delete) + end + end + + it "rejects job successfully when using perform_in" do + Sidekiq::Testing.disable! do + set = Sidekiq::ScheduledSet.new + + described_class.perform_in(30, 1) + expect(set.size).to eq(1) + + expect(described_class.perform_in(30, 1)).to be_nil + + set.each(&:delete) + end + end +end From b3255e1da4c7faba85017454376fbef3e1569783 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 26 Sep 2021 11:28:00 +0200 Subject: [PATCH 10/11] Fix return value of `base_lock` When using the `on_conflict` `replace` strategy `base_lock` has to return the return value of the conflict strategy. The replace strategy removes the previous job from the scheduled/rety set or the queue. It is then re-queued inside `lock_failed`/`call_strategy`, thus we have to return the return value of which came from `lock` inside the block of ` client_strategy.call`. --- lib/sidekiq_unique_jobs/lock/base_lock.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sidekiq_unique_jobs/lock/base_lock.rb b/lib/sidekiq_unique_jobs/lock/base_lock.rb index ef4877693..3ba5f969b 100644 --- a/lib/sidekiq_unique_jobs/lock/base_lock.rb +++ b/lib/sidekiq_unique_jobs/lock/base_lock.rb @@ -109,7 +109,6 @@ def prepare_item def lock_failed(origin: :client) reflect(:lock_failed, item) call_strategy(origin: origin) - nil end def call_strategy(origin:) From 630ac09bf718bf88a16f53b0e1bb2a279bf82050 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 26 Sep 2021 11:48:15 +0200 Subject: [PATCH 11/11] Test the `on_conflict: :replace` returns the new job id --- spec/workers/until_and_while_executing_replace_job_spec.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/workers/until_and_while_executing_replace_job_spec.rb b/spec/workers/until_and_while_executing_replace_job_spec.rb index 8e767acf8..13d889bc6 100644 --- a/spec/workers/until_and_while_executing_replace_job_spec.rb +++ b/spec/workers/until_and_while_executing_replace_job_spec.rb @@ -20,7 +20,8 @@ expect(set.size).to eq(1) expect(set.first.item["args"]).to eq(["unique", "first argument"]) - described_class.perform_at(Time.now + 30, "unique", "new argument") + job_id = described_class.perform_at(Time.now + 30, "unique", "new argument") + expect(job_id).not_to be_nil expect(set.size).to eq(1) expect(set.first.item["args"]).to eq(["unique", "new argument"]) @@ -36,7 +37,8 @@ expect(set.size).to eq(1) expect(set.first.item["args"]).to eq(["unique", "first argument"]) - described_class.perform_in(30, "unique", "new argument") + job_id = described_class.perform_in(30, "unique", "new argument") + expect(job_id).not_to be_nil expect(set.size).to eq(1) expect(set.first.item["args"]).to eq(["unique", "new argument"])