diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1396a4..46f68a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: - '3.10' - '3.11' - '3.12' + - '3.13' env: BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/rspec-${{ matrix.gemfile }}.gemfile steps: @@ -63,7 +64,7 @@ jobs: bundle exec rspec \ --format json --out tmp/rspec.json \ --format documentation && bin/check_num_tests.rb - if: ${{ matrix.ruby == 3.1 && matrix.gemfile == 3.12 }} + if: ${{ matrix.ruby == 3.1 && matrix.gemfile == 3.13 }} env: RWX_ACCESS_TOKEN: ${{ secrets.RWX_ACCESS_TOKEN }} timeout-minutes: 5 @@ -72,7 +73,7 @@ jobs: bundle exec rspec \ --format json --out tmp/rspec.json \ --format documentation && bin/check_num_tests.rb - if: ${{ matrix.ruby != 3.1 || matrix.gemfile != 3.12 }} + if: ${{ matrix.ruby != 3.1 || matrix.gemfile != 3.13 }} timeout-minutes: 5 - name: Upload Coverage if: always() diff --git a/Gemfile b/Gemfile index 88d4266..5dd8b20 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,7 @@ source "https://rubygems.org" gemspec # and the specific patch version of rspec -gem "rspec", "~> 3.12" +gem "rspec", "~> 3.13" group :development do gem "rake", "~> 13.0" diff --git a/Gemfile.lock b/Gemfile.lock index 7ebc40e..5f1b2a3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,7 +10,7 @@ PATH remote: . specs: rspec-abq (1.1.5) - rspec-core (>= 3.8.0, < 3.13.0) + rspec-core (>= 3.8.0, < 3.14.0) GEM remote: https://rubygems.org/ @@ -71,21 +71,21 @@ GEM rake (13.1.0) regexp_parser (2.9.0) rexml (3.2.6) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.3) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.4) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.7) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) + rspec-support (~> 3.13.0) rspec-retry (0.6.2) rspec-core (> 3.3) - rspec-support (3.12.2) + rspec-support (3.13.1) rubocop (1.61.0) json (~> 2.3) language_server-protocol (>= 3.17.0) @@ -167,7 +167,7 @@ DEPENDENCIES puma (~> 6.4) rack (~> 2.2) rake (~> 13.0) - rspec (~> 3.12) + rspec (~> 3.13) rspec-abq! rspec-retry (~> 0.6.2) rspec-snapshot! diff --git a/bin/generate-abq-test-output.rb b/bin/generate-abq-test-output.rb index 35cef52..cc5d22a 100755 --- a/bin/generate-abq-test-output.rb +++ b/bin/generate-abq-test-output.rb @@ -33,10 +33,10 @@ `bundle exec rspec --dry-run --format json | jq '.examples | length' > spec/NUM_TESTS` # symlink the results for the latest gemfile to the results for the default gemfile -Dir['spec/**/*-rspec-3.12.gemfile.snap'].each do |path| +Dir['spec/**/*-rspec-3.13.gemfile.snap'].each do |path| Dir.chdir(File.dirname(path)) do source = File.basename(path) - target = source.sub('rspec-3.12.gemfile', 'Gemfile') + target = source.sub('rspec-3.13.gemfile', 'Gemfile') FileUtils.rm(target) if File.exist? target FileUtils.ln_s(source, target) end diff --git a/gemfiles/rspec-3.10.gemfile.lock b/gemfiles/rspec-3.10.gemfile.lock index ef7d97c..e1b4fd3 100644 --- a/gemfiles/rspec-3.10.gemfile.lock +++ b/gemfiles/rspec-3.10.gemfile.lock @@ -10,7 +10,7 @@ PATH remote: .. specs: rspec-abq (1.1.5) - rspec-core (>= 3.8.0, < 3.13.0) + rspec-core (>= 3.8.0, < 3.14.0) GEM remote: https://rubygems.org/ diff --git a/gemfiles/rspec-3.11.gemfile.lock b/gemfiles/rspec-3.11.gemfile.lock index 08a3919..544109a 100644 --- a/gemfiles/rspec-3.11.gemfile.lock +++ b/gemfiles/rspec-3.11.gemfile.lock @@ -10,7 +10,7 @@ PATH remote: .. specs: rspec-abq (1.1.5) - rspec-core (>= 3.8.0, < 3.13.0) + rspec-core (>= 3.8.0, < 3.14.0) GEM remote: https://rubygems.org/ diff --git a/gemfiles/rspec-3.12.gemfile.lock b/gemfiles/rspec-3.12.gemfile.lock index 58fa0d5..517c82e 100644 --- a/gemfiles/rspec-3.12.gemfile.lock +++ b/gemfiles/rspec-3.12.gemfile.lock @@ -10,7 +10,7 @@ PATH remote: .. specs: rspec-abq (1.1.5) - rspec-core (>= 3.8.0, < 3.13.0) + rspec-core (>= 3.8.0, < 3.14.0) GEM remote: https://rubygems.org/ diff --git a/gemfiles/rspec-3.13.gemfile b/gemfiles/rspec-3.13.gemfile new file mode 100644 index 0000000..4532d98 --- /dev/null +++ b/gemfiles/rspec-3.13.gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in rspec-abq.gemspec +gemspec(path: "..") + +# and the specific patch version of rspec +gem "rspec", "~> 3.13", "< 3.14" +gem "rspec-snapshot", github: "rwx-research/rspec-snapshot" diff --git a/gemfiles/rspec-3.13.gemfile.lock b/gemfiles/rspec-3.13.gemfile.lock new file mode 100644 index 0000000..cd9afc5 --- /dev/null +++ b/gemfiles/rspec-3.13.gemfile.lock @@ -0,0 +1,118 @@ +GIT + remote: https://github.com/rwx-research/rspec-snapshot.git + revision: 77b3098b46190a291b32b52686d07c96cced18d5 + specs: + rspec-snapshot (2.0.1) + awesome_print (> 1.0.0) + rspec (> 3.0.0) + +PATH + remote: .. + specs: + rspec-abq (1.1.5) + rspec-core (>= 3.8.0, < 3.14.0) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) + awesome_print (1.9.2) + capybara (3.40.0) + addressable + matrix + mini_mime (>= 0.1.3) + nokogiri (~> 1.11) + rack (>= 1.6.0) + rack-test (>= 0.6.3) + regexp_parser (>= 1.5, < 3.0) + xpath (~> 3.2) + capybara-inline-screenshot (2.2.1) + capybara-screenshot + capybara-screenshot (1.0.26) + capybara (>= 1.0, < 4) + launchy + coderay (1.1.3) + diff-lcs (1.5.1) + docile (1.4.0) + launchy (2.5.2) + addressable (~> 2.8) + matrix (0.4.2) + method_source (1.0.0) + mini_mime (1.1.5) + nio4r (2.7.0) + nokogiri (1.16.2-arm64-darwin) + racc (~> 1.4) + nokogiri (1.16.2-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.16.2-x86_64-linux) + racc (~> 1.4) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + public_suffix (5.0.4) + puma (6.4.2) + nio4r (~> 2.0) + racc (1.7.3) + rack (2.2.8.1) + rack-test (2.1.0) + rack (>= 1.3) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-retry (0.6.2) + rspec-core (> 3.3) + rspec-support (3.13.1) + rubyzip (2.3.2) + selenium-webdriver (4.10.0) + rexml (~> 3.2, >= 3.2.5) + rubyzip (>= 1.2.2, < 3.0) + websocket (~> 1.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + webdrivers (5.3.1) + nokogiri (~> 1.6) + rubyzip (>= 1.3.0) + selenium-webdriver (~> 4.0, < 4.11) + websocket (1.2.10) + xpath (3.2.0) + nokogiri (~> 1.8) + +PLATFORMS + arm64-darwin-21 + arm64-darwin-23 + x86_64-darwin-21 + x86_64-linux + +DEPENDENCIES + capybara (~> 3.40) + capybara-inline-screenshot (~> 2.2.1) + nokogiri (~> 1.16) + pry (~> 0.14.1) + puma (~> 6.4) + rack (~> 2.2) + rspec (~> 3.13, < 3.14) + rspec-abq! + rspec-retry (~> 0.6.2) + rspec-snapshot! + selenium-webdriver (~> 4.10.0) + simplecov (~> 0.22.0) + webdrivers (~> 5.3) + +BUNDLED WITH + 2.3.7 diff --git a/gemfiles/rspec-3.8.gemfile.lock b/gemfiles/rspec-3.8.gemfile.lock index 99d42b4..d589cb9 100644 --- a/gemfiles/rspec-3.8.gemfile.lock +++ b/gemfiles/rspec-3.8.gemfile.lock @@ -10,7 +10,7 @@ PATH remote: .. specs: rspec-abq (1.1.5) - rspec-core (>= 3.8.0, < 3.13.0) + rspec-core (>= 3.8.0, < 3.14.0) GEM remote: https://rubygems.org/ diff --git a/gemfiles/rspec-3.9.gemfile.lock b/gemfiles/rspec-3.9.gemfile.lock index 2e7f5b9..b9cef14 100644 --- a/gemfiles/rspec-3.9.gemfile.lock +++ b/gemfiles/rspec-3.9.gemfile.lock @@ -10,7 +10,7 @@ PATH remote: .. specs: rspec-abq (1.1.5) - rspec-core (>= 3.8.0, < 3.13.0) + rspec-core (>= 3.8.0, < 3.14.0) GEM remote: https://rubygems.org/ diff --git a/lib/rspec/abq/extensions.rb b/lib/rspec/abq/extensions.rb index 9e0d98e..65c3e5f 100644 --- a/lib/rspec/abq/extensions.rb +++ b/lib/rspec/abq/extensions.rb @@ -119,6 +119,10 @@ module Runner # or the configured failure exit status (1 by default) if specs # failed. def run_specs(example_groups) + if Gem::Version.new(RSpec::Core::Version::STRING) >= Gem::Version.new("3.11.0") && RSpec.world.rspec_is_quitting + return exit_code(false) + end + if !!ENV[ABQ_GENERATE_MANIFEST] # before abq can start workers, it asks for a manifest RSpec::Abq::Manifest.write_manifest(example_groups, RSpec.configuration.seed, RSpec.configuration.ordering_registry) diff --git a/rspec-abq.gemspec b/rspec-abq.gemspec index 43f739a..3305884 100644 --- a/rspec-abq.gemspec +++ b/rspec-abq.gemspec @@ -27,7 +27,7 @@ Gem::Specification.new do |spec| spec.files = `git ls-files -- lib/*`.split("\n") + %w[README.md LICENSE.md] spec.require_paths = ["lib"] - spec.add_dependency "rspec-core", ">= 3.8.0", "< 3.13.0" + spec.add_dependency "rspec-core", ">= 3.8.0", "< 3.14.0" spec.add_development_dependency "pry", "~> 0.14.1" spec.add_development_dependency "rspec-retry", "~> 0.6.2" spec.add_development_dependency "capybara", "~> 3.40" diff --git a/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-Gemfile.snap index 5a121b2..2375b8c 120000 --- a/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..c2d2b09 --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-passed-as-CLI-argument-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,141 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 0, + "otherErrors": 0, + "pended": 3, + "quarantined": 0, + "retries": 0, + "skipped": 0, + "status": { + "kind": "successful" + }, + "successful": 1, + "tests": 4, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/successful_specs.rb[1:1]", + "lineage": [ + "a successful group", + "has a successful test" + ], + "location": { + "file": "./spec/fixture_specs/successful_specs.rb", + "line": 299 + }, + "name": "a successful group has a successful test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:3:1]", + "lineage": [ + "pending tests", + "a pending group via a tag", + "is pending despite not being marked as pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests a pending group via a tag is pending despite not being marked as pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:2]", + "lineage": [ + "pending tests", + "has a pending test with a tag" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with a tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:1]", + "lineage": [ + "pending tests", + "has a pending test with pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with pending" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-Gemfile.snap index 008ae9b..909ac46 120000 --- a/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..0412698 --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-for-random-ordering-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,84 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 0, + "otherErrors": 0, + "pended": 1, + "quarantined": 0, + "retries": 0, + "skipped": 0, + "status": { + "kind": "successful" + }, + "successful": 1, + "tests": 2, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_that_sets_up_random_ordering.rb[1:2]", + "lineage": [ + "random ordering", + "is pended" + ], + "location": { + "file": "./spec/fixture_specs/spec_that_sets_up_random_ordering.rb", + "line": 299 + }, + "name": "random ordering is pended" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_that_sets_up_random_ordering.rb[1:1]", + "lineage": [ + "random ordering", + "passes" + ], + "location": { + "file": "./spec/fixture_specs/spec_that_sets_up_random_ordering.rb", + "line": 299 + }, + "name": "random ordering passes" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-Gemfile.snap index fb21e53..52bc7fb 120000 --- a/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-for-random-seed-set-in-rspec-config-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-for-random-seed-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..5b384c4 --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-for-random-seed-set-in-rspec-config-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,84 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 0, + "otherErrors": 0, + "pended": 1, + "quarantined": 0, + "retries": 0, + "skipped": 0, + "status": { + "kind": "successful" + }, + "successful": 1, + "tests": 2, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_that_sets_up_random_seed.rb[1:2]", + "lineage": [ + "random ordering", + "is pended" + ], + "location": { + "file": "./spec/fixture_specs/spec_that_sets_up_random_seed.rb", + "line": 299 + }, + "name": "random ordering is pended" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_that_sets_up_random_seed.rb[1:1]", + "lineage": [ + "random ordering", + "passes" + ], + "location": { + "file": "./spec/fixture_specs/spec_that_sets_up_random_seed.rb", + "line": 299 + }, + "name": "random ordering passes" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-Gemfile.snap index a0b7573..2e3e2b6 120000 --- a/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..ade084e --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-for-specs-together-with-a-hardcoded-seed-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,529 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 5, + "otherErrors": 0, + "pended": 3, + "quarantined": 0, + "retries": 0, + "skipped": 6, + "status": { + "kind": "failed" + }, + "successful": 1, + "tests": 15, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a failing group failing in a before block is failing despite being successful\n Failure/Error: before(:all) { fail('because') }\n\n RuntimeError:\n because\n # ./spec/fixture_specs/failing_specs.rb:0:in `block (3 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:2:1]", + "lineage": [ + "a failing group", + "failing in a before block", + "is failing despite being successful" + ], + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 299 + }, + "name": "a failing group failing in a before block is failing despite being successful" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RSpec::Expectations::ExpectationNotMetError", + "kind": "failed", + "message": "\n 1) a failing group has a failing test\n Failure/Error: expect(false).to eq(true)\n\n expected: true\n got: false\n\n (compared using ==)\n\n Diff:\n @@ -1 +1 @@\n -true\n +false\n # ./spec/fixture_specs/failing_specs.rb:0:in `block (2 levels) in '\n # ./spec/spec_helper.rb:0:in `block (2 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `each'\n # ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:1]", + "lineage": [ + "a failing group", + "has a failing test" + ], + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 299 + }, + "name": "a failing group has a failing test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group has a raising test\n Failure/Error: raise 'the roof'\n\n RuntimeError:\n the roof\n # ./spec/fixture_specs/raising_specs.rb:0:in `block (2 levels) in '\n # ./spec/spec_helper.rb:0:in `block (2 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `each'\n # ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:1]", + "lineage": [ + "a raising group", + "has a raising test" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group has a raising test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group it can handle an exception in a before all hook should also fail\n Failure/Error: raise \"raising from before(:all)\"\n\n RuntimeError:\n raising from before(:all)\n # ./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:2]", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should also fail" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group it can handle an exception in a before all hook should also fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group it can handle an exception in a before all hook should fail\n Failure/Error: raise \"raising from before(:all)\"\n\n RuntimeError:\n raising from before(:all)\n # ./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:1]", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should fail" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group it can handle an exception in a before all hook should fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/successful_specs.rb[1:1]", + "lineage": [ + "a successful group", + "has a successful test" + ], + "location": { + "file": "./spec/fixture_specs/successful_specs.rb", + "line": 299 + }, + "name": "a successful group has a successful test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:3:1]", + "lineage": [ + "pending tests", + "a pending group via a tag", + "is pending despite not being marked as pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests a pending group via a tag is pending despite not being marked as pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:2]", + "lineage": [ + "pending tests", + "has a pending test with a tag" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with a tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:1]", + "lineage": [ + "pending tests", + "has a pending test with pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:4:1]", + "lineage": [ + "skipped tests", + "a skipped group via tag", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests a skipped group via tag is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:2]", + "lineage": [ + "skipped tests", + "has a skipped test with skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests has a skipped test with skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "skip": "Temporarily skipped with xdescribe" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:5:1]", + "lineage": [ + "skipped tests", + "skipped group with xdescribe", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped group with xdescribe is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:6:1]", + "lineage": [ + "skipped tests", + "skipped in a before block", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped in a before block is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:3]", + "lineage": [ + "skipped tests", + "skipped test with tag" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped test with tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "skip": "Temporarily skipped with xit" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:1]", + "lineage": [ + "skipped tests", + "skipped test with xit" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped test with xit" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-test-results-Gemfile.snap index 9814996..64bedaf 120000 --- a/spec/features/__snapshots__/has-consistent-output-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..048f351 --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,529 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 5, + "otherErrors": 0, + "pended": 3, + "quarantined": 0, + "retries": 0, + "skipped": 6, + "status": { + "kind": "failed" + }, + "successful": 1, + "tests": 15, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a failing group failing in a before block is failing despite being successful\n \u001b[31mFailure/Error: \u001b[0m\u001b[32mbefore\u001b[0m(\u001b[33m:all\u001b[0m) { fail(\u001b[31m\u001b[1;31m'\u001b[0m\u001b[31mbecause\u001b[1;31m'\u001b[0m\u001b[31m\u001b[0m) }\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31mRuntimeError:\u001b[0m\n \u001b[31m because\u001b[0m\n \u001b[36m# ./spec/fixture_specs/failing_specs.rb:0:in `block (3 levels) in '\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_specs'\u001b[0m\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:2:1]", + "lineage": [ + "a failing group", + "failing in a before block", + "is failing despite being successful" + ], + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 299 + }, + "name": "a failing group failing in a before block is failing despite being successful" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RSpec::Expectations::ExpectationNotMetError", + "kind": "failed", + "message": "\n 1) a failing group has a failing test\n \u001b[31mFailure/Error: \u001b[0m\u001b[32mexpect\u001b[0m(\u001b[1;36mfalse\u001b[0m).to eq(\u001b[1;36mtrue\u001b[0m)\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31m expected: true\u001b[0m\n \u001b[31m got: false\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31m (compared using ==)\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31m Diff:\u001b[0m\u001b[0m\n \u001b[31m \u001b[0m\u001b[34m@@ -1 +1 @@\u001b[0m\n \u001b[31m \u001b[0m\u001b[31m-true\u001b[0m\n \u001b[31m \u001b[0m\u001b[32m+false\u001b[0m\n \u001b[31m \u001b[0m\u001b[0m\n \u001b[36m# ./spec/fixture_specs/failing_specs.rb:0:in `block (2 levels) in '\u001b[0m\n \u001b[36m# ./spec/spec_helper.rb:0:in `block (2 levels) in '\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `each'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_specs'\u001b[0m\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:1]", + "lineage": [ + "a failing group", + "has a failing test" + ], + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 299 + }, + "name": "a failing group has a failing test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group has a raising test\n \u001b[31mFailure/Error: \u001b[0mraise \u001b[31m\u001b[1;31m'\u001b[0m\u001b[31mthe roof\u001b[1;31m'\u001b[0m\u001b[31m\u001b[0m\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31mRuntimeError:\u001b[0m\n \u001b[31m the roof\u001b[0m\n \u001b[36m# ./spec/fixture_specs/raising_specs.rb:0:in `block (2 levels) in '\u001b[0m\n \u001b[36m# ./spec/spec_helper.rb:0:in `block (2 levels) in '\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `each'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_specs'\u001b[0m\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:1]", + "lineage": [ + "a raising group", + "has a raising test" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group has a raising test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group it can handle an exception in a before all hook should also fail\n \u001b[31mFailure/Error: \u001b[0mraise \u001b[31m\u001b[1;31m\"\u001b[0m\u001b[31mraising from before(:all)\u001b[1;31m\"\u001b[0m\u001b[31m\u001b[0m\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31mRuntimeError:\u001b[0m\n \u001b[31m raising from before(:all)\u001b[0m\n \u001b[36m# ./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_specs'\u001b[0m\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:2]", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should also fail" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group it can handle an exception in a before all hook should also fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group it can handle an exception in a before all hook should fail\n \u001b[31mFailure/Error: \u001b[0mraise \u001b[31m\u001b[1;31m\"\u001b[0m\u001b[31mraising from before(:all)\u001b[1;31m\"\u001b[0m\u001b[31m\u001b[0m\u001b[0m\n \u001b[31m\u001b[0m\n \u001b[31mRuntimeError:\u001b[0m\n \u001b[31m raising from before(:all)\u001b[0m\n \u001b[36m# ./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_specs'\u001b[0m\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:1]", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should fail" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group it can handle an exception in a before all hook should fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/successful_specs.rb[1:1]", + "lineage": [ + "a successful group", + "has a successful test" + ], + "location": { + "file": "./spec/fixture_specs/successful_specs.rb", + "line": 299 + }, + "name": "a successful group has a successful test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:3:1]", + "lineage": [ + "pending tests", + "a pending group via a tag", + "is pending despite not being marked as pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests a pending group via a tag is pending despite not being marked as pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:2]", + "lineage": [ + "pending tests", + "has a pending test with a tag" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with a tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:1]", + "lineage": [ + "pending tests", + "has a pending test with pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:4:1]", + "lineage": [ + "skipped tests", + "a skipped group via tag", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests a skipped group via tag is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:2]", + "lineage": [ + "skipped tests", + "has a skipped test with skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests has a skipped test with skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "skip": "Temporarily skipped with xdescribe" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:5:1]", + "lineage": [ + "skipped tests", + "skipped group with xdescribe", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped group with xdescribe is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:6:1]", + "lineage": [ + "skipped tests", + "skipped in a before block", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped in a before block is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:3]", + "lineage": [ + "skipped tests", + "skipped test with tag" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped test with tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "skip": "Temporarily skipped with xit" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:1]", + "lineage": [ + "skipped tests", + "skipped test with xit" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped test with xit" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-Gemfile.snap index a6010c9..b33c9be 120000 --- a/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..8228eaa --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-with-capybara-&-capybara-inline-screenshot-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,105 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 1, + "otherErrors": 0, + "pended": 0, + "quarantined": 0, + "retries": 0, + "skipped": 0, + "status": { + "kind": "failed" + }, + "successful": 1, + "tests": 2, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "screenshot": { + "html": "tmp/screenshot.html", + "image": "tmp/screenshot.png" + }, + "type": "feature" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/spec_with_capybara.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RSpec::Expectations::ExpectationNotMetError", + "kind": "failed", + "message": "\n 1) a spec with capybara can fail\n \u001b[31mFailure/Error: \u001b[0m\u001b[32mexpect\u001b[0m(page).to have_content(\u001b[31m\u001b[1;31m\"\u001b[0m\u001b[31mA huge website\u001b[1;31m\"\u001b[0m\u001b[31m\u001b[0m)\u001b[0m\n \u001b[31m expected to find text \"A huge website\" in \"A tiny website\"\u001b[0m\n \u001b[36m# ./spec/fixture_specs/spec_with_capybara.rb:0:in `block (2 levels) in '\u001b[0m\n \u001b[36m# ./spec/spec_helper.rb:0:in `block (2 levels) in '\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `each'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `map'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\u001b[0m\n \u001b[36m# ./lib/rspec/abq/extensions.rb:0:in `run_specs'\u001b[0m\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_with_capybara.rb[1:2]", + "lineage": [ + "a spec with capybara", + "can fail" + ], + "location": { + "file": "./spec/fixture_specs/spec_with_capybara.rb", + "line": 299 + }, + "name": "a spec with capybara can fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "type": "feature" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_with_capybara.rb[1:1]", + "lineage": [ + "a spec with capybara", + "can succeed" + ], + "location": { + "file": "./spec/fixture_specs/spec_with_capybara.rb", + "line": 299 + }, + "name": "a spec with capybara can succeed" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-Gemfile.snap index cfdb428..3523c5b 120000 --- a/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-with-capybara-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-with-capybara-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..dbadc04 --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-with-capybara-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,101 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 1, + "otherErrors": 0, + "pended": 0, + "quarantined": 0, + "retries": 0, + "skipped": 0, + "status": { + "kind": "failed" + }, + "successful": 1, + "tests": 2, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "type": "feature" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/spec_with_capybara.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RSpec::Expectations::ExpectationNotMetError", + "kind": "failed", + "message": "\n 1) a spec with capybara can fail\n Failure/Error: expect(page).to have_content(\"A huge website\")\n expected to find text \"A huge website\" in \"A tiny website\"\n # ./spec/fixture_specs/spec_with_capybara.rb:0:in `block (2 levels) in '\n # ./spec/spec_helper.rb:0:in `block (2 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `each'\n # ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_with_capybara.rb[1:2]", + "lineage": [ + "a spec with capybara", + "can fail" + ], + "location": { + "file": "./spec/fixture_specs/spec_with_capybara.rb", + "line": 299 + }, + "name": "a spec with capybara can fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "type": "feature" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/spec_with_capybara.rb[1:1]", + "lineage": [ + "a spec with capybara", + "can succeed" + ], + "location": { + "file": "./spec/fixture_specs/spec_with_capybara.rb", + "line": 299 + }, + "name": "a spec with capybara can succeed" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-Gemfile.snap b/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-Gemfile.snap index 5fb9222..2e25bdd 120000 --- a/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-Gemfile.snap +++ b/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-Gemfile.snap @@ -1 +1 @@ -has-consistent-output-with-rspec-retry-test-results-rspec-3.12.gemfile.snap \ No newline at end of file +has-consistent-output-with-rspec-retry-test-results-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..2c267db --- /dev/null +++ b/spec/features/__snapshots__/has-consistent-output-with-rspec-retry-test-results-rspec-3.13.gemfile.snap @@ -0,0 +1,589 @@ +{ + "$schema": "https://raw.githubusercontent.com/rwx-research/test-results-schema/main/v1.json", + "framework": { + "kind": "RSpec", + "language": "Ruby" + }, + "otherErrors": [ + + ], + "summary": { + "canceled": 0, + "failed": 5, + "otherErrors": 0, + "pended": 3, + "quarantined": 0, + "retries": 2, + "skipped": 6, + "status": { + "kind": "failed" + }, + "successful": 1, + "tests": 15, + "timedOut": 0, + "todo": 0 + }, + "tests": [ + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a failing group failing in a before block is failing despite being successful\n Failure/Error: before(:all) { fail('because') }\n\n RuntimeError:\n because\n # ./spec/fixture_specs/failing_specs.rb:0:in `block (3 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:2:1]", + "lineage": [ + "a failing group", + "failing in a before block", + "is failing despite being successful" + ], + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 299 + }, + "name": "a failing group failing in a before block is failing despite being successful" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 1, + "retry_exceptions": [ + "\nexpected: true\n got: false\n\n(compared using ==)\n\nDiff:\n@@ -1 +1 @@\n-true\n+false\n", + "\nexpected: true\n got: false\n\n(compared using ==)\n\nDiff:\n@@ -1 +1 @@\n-true\n+false\n" + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RSpec::Expectations::ExpectationNotMetError", + "kind": "failed", + "message": "\n 1) a failing group has a failing test\n Failure/Error: expect(false).to eq(true)\n\n expected: true\n got: false\n\n (compared using ==)\n\n Diff:\n @@ -1 +1 @@\n -true\n +false\n # ./spec/fixture_specs/failing_specs.rb:0:in `block (2 levels) in '\n # ./spec/spec_helper.rb:0:in `block (2 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `each'\n # ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:1]", + "lineage": [ + "a failing group", + "has a failing test" + ], + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 299 + }, + "name": "a failing group has a failing test", + "pastAttempts": [ + { + "durationInNanoseconds": 0, + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "status": { + "kind": "failed", + "message": "\nexpected: true\n got: false\n\n(compared using ==)\n\nDiff:\n@@ -1 +1 @@\n-true\n+false\n" + } + } + ] + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 1, + "retry_exceptions": [ + "the roof", + "the roof" + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (2 levels) in '", + "./spec/spec_helper.rb:0:in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `each'", + "./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group has a raising test\n Failure/Error: raise 'the roof'\n\n RuntimeError:\n the roof\n # ./spec/fixture_specs/raising_specs.rb:0:in `block (2 levels) in '\n # ./spec/spec_helper.rb:0:in `block (2 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `each'\n # ./lib/rspec/abq/extensions.rb:0:in `run_examples_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:1]", + "lineage": [ + "a raising group", + "has a raising test" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group has a raising test", + "pastAttempts": [ + { + "durationInNanoseconds": 0, + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "status": { + "kind": "failed", + "message": "the roof" + } + } + ] + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group it can handle an exception in a before all hook should also fail\n Failure/Error: raise \"raising from before(:all)\"\n\n RuntimeError:\n raising from before(:all)\n # ./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:2]", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should also fail" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group it can handle an exception in a before all hook should also fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `run_with_abq'", + "./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `map'", + "./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `block in run_specs'", + "./lib/rspec/abq/extensions.rb:0:in `run_specs'" + ], + "exception": "RuntimeError", + "kind": "failed", + "message": "\n 1) a raising group it can handle an exception in a before all hook should fail\n Failure/Error: raise \"raising from before(:all)\"\n\n RuntimeError:\n raising from before(:all)\n # ./spec/fixture_specs/raising_specs.rb:0:in `block (3 levels) in '\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `run_with_abq'\n # ./lib/rspec/abq/extensions.rb:0:in `block (3 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `map'\n # ./lib/rspec/abq/extensions.rb:0:in `block (2 levels) in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `block in run_specs'\n # ./lib/rspec/abq/extensions.rb:0:in `run_specs'\n" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:1]", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should fail" + ], + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 299 + }, + "name": "a raising group it can handle an exception in a before all hook should fail" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 0, + "retry_exceptions": [ + + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "successful" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/successful_specs.rb[1:1]", + "lineage": [ + "a successful group", + "has a successful test" + ], + "location": { + "file": "./spec/fixture_specs/successful_specs.rb", + "line": 299 + }, + "name": "a successful group has a successful test" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 0, + "retry_exceptions": [ + + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:3:1]", + "lineage": [ + "pending tests", + "a pending group via a tag", + "is pending despite not being marked as pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests a pending group via a tag is pending despite not being marked as pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 0, + "retry_exceptions": [ + + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:2]", + "lineage": [ + "pending tests", + "has a pending test with a tag" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with a tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 0, + "retry_exceptions": [ + + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "pended" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:1]", + "lineage": [ + "pending tests", + "has a pending test with pending" + ], + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 299 + }, + "name": "pending tests has a pending test with pending" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:4:1]", + "lineage": [ + "skipped tests", + "a skipped group via tag", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests a skipped group via tag is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "retry_attempts": 0, + "retry_exceptions": [ + + ] + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:2]", + "lineage": [ + "skipped tests", + "has a skipped test with skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests has a skipped test with skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "skip": "Temporarily skipped with xdescribe" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:5:1]", + "lineage": [ + "skipped tests", + "skipped group with xdescribe", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped group with xdescribe is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:6:1]", + "lineage": [ + "skipped tests", + "skipped in a before block", + "is skipped despite not being marked as skip" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped in a before block is skipped despite not being marked as skip" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + } + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:3]", + "lineage": [ + "skipped tests", + "skipped test with tag" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped test with tag" + }, + { + "attempt": { + "durationInNanoseconds": 234000, + "finishedAt": "2023-01-01T00:00:00Z", + "meta": { + "abq_metadata": { + "runner": 1, + "worker": 0 + }, + "skip": "Temporarily skipped with xit" + }, + "startedAt": "2023-01-01T00:00:00Z", + "status": { + "kind": "skipped" + }, + "stderr": "redacted", + "stdout": "redacted" + }, + "id": "./spec/fixture_specs/skipped_specs.rb[1:1]", + "lineage": [ + "skipped tests", + "skipped test with xit" + ], + "location": { + "file": "./spec/fixture_specs/skipped_specs.rb", + "line": 299 + }, + "name": "skipped tests skipped test with xit" + } + ] +} \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_failing_specs-Gemfile.snap b/spec/features/__snapshots__/test_results_failing_specs-Gemfile.snap index 0a6ab02..a4e2b6b 120000 --- a/spec/features/__snapshots__/test_results_failing_specs-Gemfile.snap +++ b/spec/features/__snapshots__/test_results_failing_specs-Gemfile.snap @@ -1 +1 @@ -test_results_failing_specs-rspec-3.12.gemfile.snap \ No newline at end of file +test_results_failing_specs-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_failing_specs-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/test_results_failing_specs-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..ef0ee80 --- /dev/null +++ b/spec/features/__snapshots__/test_results_failing_specs-rspec-3.13.gemfile.snap @@ -0,0 +1,82 @@ +[ + { + "test_result": { + "status": { + "type": "failure", + "exception": "RSpec::Expectations::ExpectationNotMetError", + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb::in `block (2 levels) in '", + "./spec/spec_helper.rb::in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb::in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb::in `each'", + "./lib/rspec/abq/extensions.rb::in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `block in run_specs'", + "./lib/rspec/abq/extensions.rb::in `run_specs'" + ] + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:1]", + "display_name": "a failing group has a failing test", + "output": "\n1) a failing group has a failing test\nFailure/Error: expect(false).to eq(true)\nexpected: true\ngot: false\n(compared using ==)\nDiff:\n@@ -1 +1 @@\n-true\n+false\n# ./spec/fixture_specs/failing_specs.rb::in `block (2 levels) in '\n# ./spec/spec_helper.rb::in `block (2 levels) in '\n# ./lib/rspec/abq/extensions.rb::in `block in run_examples_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `each'\n# ./lib/rspec/abq/extensions.rb::in `run_examples_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `block in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `run_specs'\n", + "runtime": "", + "tags": [ + + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 5 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "a failing group", + "has a failing test" + ] + } + }, + { + "test_result": { + "status": { + "type": "error", + "exception": "RuntimeError", + "backtrace": [ + "./spec/fixture_specs/failing_specs.rb::in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `block in run_specs'", + "./lib/rspec/abq/extensions.rb::in `run_specs'" + ] + }, + "id": "./spec/fixture_specs/failing_specs.rb[1:2:1]", + "display_name": "a failing group failing in a before block is failing despite being successful", + "output": "\n1) a failing group failing in a before block is failing despite being successful\nFailure/Error: before(:all) { fail('because') }\nRuntimeError:\nbecause\n# ./spec/fixture_specs/failing_specs.rb::in `block (3 levels) in '\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block in run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `block in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `run_specs'\n", + "runtime": "", + "tags": [ + + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/failing_specs.rb", + "line": 12 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "a failing group", + "failing in a before block", + "is failing despite being successful" + ] + } + } +] \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_pending_specs-Gemfile.snap b/spec/features/__snapshots__/test_results_pending_specs-Gemfile.snap index e50ed0c..01a4149 120000 --- a/spec/features/__snapshots__/test_results_pending_specs-Gemfile.snap +++ b/spec/features/__snapshots__/test_results_pending_specs-Gemfile.snap @@ -1 +1 @@ -test_results_pending_specs-rspec-3.12.gemfile.snap \ No newline at end of file +test_results_pending_specs-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_pending_specs-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/test_results_pending_specs-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..9f43e4c --- /dev/null +++ b/spec/features/__snapshots__/test_results_pending_specs-rspec-3.13.gemfile.snap @@ -0,0 +1,81 @@ +[ + { + "test_result": { + "status": { + "type": "pending" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:1]", + "display_name": "pending tests has a pending test with pending", + "output": null, + "runtime": "", + "tags": [ + "pending" + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 5 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "pending tests", + "has a pending test with pending" + ] + } + }, + { + "test_result": { + "status": { + "type": "pending" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:2]", + "display_name": "pending tests has a pending test with a tag", + "output": null, + "runtime": "", + "tags": [ + "pending" + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 10 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "pending tests", + "has a pending test with a tag" + ] + } + }, + { + "test_result": { + "status": { + "type": "pending" + }, + "id": "./spec/fixture_specs/pending_specs.rb[1:3:1]", + "display_name": "pending tests a pending group via a tag is pending despite not being marked as pending", + "output": null, + "runtime": "", + "tags": [ + "pending" + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/pending_specs.rb", + "line": 15 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "pending tests", + "a pending group via a tag", + "is pending despite not being marked as pending" + ] + } + } +] \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_raising_specs-Gemfile.snap b/spec/features/__snapshots__/test_results_raising_specs-Gemfile.snap index 3223734..c0becaa 120000 --- a/spec/features/__snapshots__/test_results_raising_specs-Gemfile.snap +++ b/spec/features/__snapshots__/test_results_raising_specs-Gemfile.snap @@ -1 +1 @@ -test_results_raising_specs-rspec-3.12.gemfile.snap \ No newline at end of file +test_results_raising_specs-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_raising_specs-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/test_results_raising_specs-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..871a97c --- /dev/null +++ b/spec/features/__snapshots__/test_results_raising_specs-rspec-3.13.gemfile.snap @@ -0,0 +1,122 @@ +[ + { + "test_result": { + "status": { + "type": "error", + "exception": "RuntimeError", + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb::in `block (2 levels) in '", + "./spec/spec_helper.rb::in `block (2 levels) in '", + "./lib/rspec/abq/extensions.rb::in `block in run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb::in `each'", + "./lib/rspec/abq/extensions.rb::in `run_examples_with_abq'", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `block in run_specs'", + "./lib/rspec/abq/extensions.rb::in `run_specs'" + ] + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:1]", + "display_name": "a raising group has a raising test", + "output": "\n1) a raising group has a raising test\nFailure/Error: raise 'the roof'\nRuntimeError:\nthe roof\n# ./spec/fixture_specs/raising_specs.rb::in `block (2 levels) in '\n# ./spec/spec_helper.rb::in `block (2 levels) in '\n# ./lib/rspec/abq/extensions.rb::in `block in run_examples_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `each'\n# ./lib/rspec/abq/extensions.rb::in `run_examples_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `block in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `run_specs'\n", + "runtime": "", + "tags": [ + + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 5 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "a raising group", + "has a raising test" + ] + } + }, + { + "test_result": { + "status": { + "type": "error", + "exception": "RuntimeError", + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb::in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `block in run_specs'", + "./lib/rspec/abq/extensions.rb::in `run_specs'" + ] + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:1]", + "display_name": "a raising group it can handle an exception in a before all hook should fail", + "output": "\n1) a raising group it can handle an exception in a before all hook should fail\nFailure/Error: raise \"raising from before(:all)\"\nRuntimeError:\nraising from before(:all)\n# ./spec/fixture_specs/raising_specs.rb::in `block (3 levels) in '\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block in run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `block in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `run_specs'\n", + "runtime": "", + "tags": [ + + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 14 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should fail" + ] + } + }, + { + "test_result": { + "status": { + "type": "error", + "exception": "RuntimeError", + "backtrace": [ + "./spec/fixture_specs/raising_specs.rb::in `block (3 levels) in '", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block in run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `run_with_abq'", + "./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `map'", + "./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'", + "./lib/rspec/abq/extensions.rb::in `block in run_specs'", + "./lib/rspec/abq/extensions.rb::in `run_specs'" + ] + }, + "id": "./spec/fixture_specs/raising_specs.rb[1:2:2]", + "display_name": "a raising group it can handle an exception in a before all hook should also fail", + "output": "\n1) a raising group it can handle an exception in a before all hook should also fail\nFailure/Error: raise \"raising from before(:all)\"\nRuntimeError:\nraising from before(:all)\n# ./spec/fixture_specs/raising_specs.rb::in `block (3 levels) in '\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block in run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `run_with_abq'\n# ./lib/rspec/abq/extensions.rb::in `block (3 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `map'\n# ./lib/rspec/abq/extensions.rb::in `block (2 levels) in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `block in run_specs'\n# ./lib/rspec/abq/extensions.rb::in `run_specs'\n", + "runtime": "", + "tags": [ + + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/raising_specs.rb", + "line": 19 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "a raising group", + "it can handle an exception in a before all hook", + "should also fail" + ] + } + } +] \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_successful_specs-Gemfile.snap b/spec/features/__snapshots__/test_results_successful_specs-Gemfile.snap index 750c895..a361031 120000 --- a/spec/features/__snapshots__/test_results_successful_specs-Gemfile.snap +++ b/spec/features/__snapshots__/test_results_successful_specs-Gemfile.snap @@ -1 +1 @@ -test_results_successful_specs-rspec-3.12.gemfile.snap \ No newline at end of file +test_results_successful_specs-rspec-3.13.gemfile.snap \ No newline at end of file diff --git a/spec/features/__snapshots__/test_results_successful_specs-rspec-3.13.gemfile.snap b/spec/features/__snapshots__/test_results_successful_specs-rspec-3.13.gemfile.snap new file mode 100644 index 0000000..8b1f8f4 --- /dev/null +++ b/spec/features/__snapshots__/test_results_successful_specs-rspec-3.13.gemfile.snap @@ -0,0 +1,28 @@ +[ + { + "test_result": { + "status": { + "type": "success" + }, + "id": "./spec/fixture_specs/successful_specs.rb[1:1]", + "display_name": "a successful group has a successful test", + "output": null, + "runtime": "", + "tags": [ + + ], + "meta": { + }, + "location": { + "file": "./spec/fixture_specs/successful_specs.rb", + "line": 5 + }, + "started_at": "", + "finished_at": "", + "lineage": [ + "a successful group", + "has a successful test" + ] + } + } +] \ No newline at end of file