diff --git a/CHANGELOG.md b/CHANGELOG.md index 0307bdf..57fab31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning]. ## [Unreleased] +### Added + +- Add coverage for tested API operations. ([@skryukov]) + + ```ruby + + # spec/rails_helper.rb + + RSpec.configure do |config| + # To enable coverage, pass `coverage: :report` option, + # and to raise an error when an operation is not covered, pass `coverage: :strict` option: + config.include Skooma::RSpec[Rails.root.join("docs", "openapi.yml"), coverage: :report], type: :request + end + ``` + + ```shell + $ bundle exec rspec + # ... + OpenAPI schema /openapi.yml coverage report: 110 / 194 operations (56.7%) covered. + Uncovered paths: + GET /api/uncovered 200 + GET /api/partially_covered 403 + # ... + ``` + ## [0.3.0] - 2024-04-09 ### Changed @@ -39,16 +64,16 @@ and this project adheres to [Semantic Versioning]. - Add support for APIs mounted under a path prefix. ([@skryukov]) -```ruby -# spec/rails_helper.rb - -RSpec.configure do |config| - # ... - path_to_openapi = Rails.root.join("docs", "openapi.yml") - # pass path_prefix option if your API is mounted under a prefix: - config.include Skooma::RSpec[path_to_openapi, path_prefix: "/internal/api"], type: :request -end -``` + ```ruby + # spec/rails_helper.rb + + RSpec.configure do |config| + # ... + path_to_openapi = Rails.root.join("docs", "openapi.yml") + # pass path_prefix option if your API is mounted under a prefix: + config.include Skooma::RSpec[path_to_openapi, path_prefix: "/internal/api"], type: :request + end + ``` ### Changed diff --git a/README.md b/README.md index 8c069b6..361a118 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,10 @@ RSpec.configure do |config| # OR pass path_prefix option if your API is mounted under a prefix: config.include Skooma::RSpec[path_to_openapi, path_prefix: "/internal/api"], type: :request + + # To enable coverage, pass `coverage: :report` option, + # and to raise an error when an operation is not covered, pass `coverage: :strict` option: + config.include Skooma::RSpec[path_to_openapi, coverage: :report], type: :request end ``` diff --git a/examples/minitest.rb b/examples/minitest.rb index d488b66..a8b7a8f 100644 --- a/examples/minitest.rb +++ b/examples/minitest.rb @@ -18,7 +18,8 @@ describe TestApp do include Rack::Test::Methods - include Skooma::Minitest[File.join(__dir__, "openapi.yml")] + + include Skooma::Minitest[File.join(__dir__, "openapi.yml"), coverage: :report] def app TestApp["bar"] diff --git a/examples/rails_app/spec/rails_helper.rb b/examples/rails_app/spec/rails_helper.rb index 7fb3d4d..007608e 100644 --- a/examples/rails_app/spec/rails_helper.rb +++ b/examples/rails_app/spec/rails_helper.rb @@ -16,6 +16,6 @@ # You can use different RSpec filters if you want to test different API descriptions. # Check RSpec's config.define_derived_metadata for better UX. - config.include Skooma::RSpec[bar_openapi, path_prefix: "/bar"], :bar_api - config.include Skooma::RSpec[baz_openapi, path_prefix: "/baz"], :baz_api + config.include Skooma::RSpec[bar_openapi, path_prefix: "/bar", coverage: :strict], :bar_api + config.include Skooma::RSpec[baz_openapi, path_prefix: "/baz", coverage: :strict], :baz_api end diff --git a/examples/rspec.rb b/examples/rspec.rb index 360118e..58d23e9 100644 --- a/examples/rspec.rb +++ b/examples/rspec.rb @@ -17,7 +17,7 @@ RSpec.configure do |config| path_to_openapi = File.join(__dir__, "openapi.yml") - config.include Skooma::RSpec[path_to_openapi], type: :request + config.include Skooma::RSpec[path_to_openapi, coverage: :strict], type: :request config.include Rack::Test::Methods, type: :request end diff --git a/lib/skooma/coverage.rb b/lib/skooma/coverage.rb new file mode 100644 index 0000000..d00ec30 --- /dev/null +++ b/lib/skooma/coverage.rb @@ -0,0 +1,90 @@ +module Skooma + class NoopCoverage + def track_request(*) + end + + def report + end + end + + class Coverage + class SimpleReport + def initialize(coverage) + @coverage = coverage + end + + attr_reader :coverage + + def report + puts <<~MSG + OpenAPI schema #{URI.parse(coverage.schema.uri.to_s).path} coverage report: #{coverage.covered_paths.count} / #{coverage.defined_paths.count} operations (#{coverage.covered_percent.round(2)}%) covered. + #{coverage.uncovered_paths.empty? ? "All paths are covered!" : "Uncovered paths:"} + #{coverage.uncovered_paths.map { |method, path, status| "#{method.upcase} #{path} #{status}" }.join("\n")} + MSG + end + end + + def self.new(schema, mode: nil, format: nil) + case mode + when nil, false + NoopCoverage.new + when :report, :strict + super + else + raise ArgumentError, "Invalid coverage: #{mode}, expected :report, :strict, or false" + end + end + + attr_reader :mode, :format, :defined_paths, :covered_paths, :schema + + def initialize(schema, mode:, format:) + @schema = schema + @mode = mode + @format = format || SimpleReport + @defined_paths = find_defined_paths(schema) + @covered_paths = Set.new + end + + def track_request(result) + operation = [nil, nil, nil] + result.collect_annotations(result.instance, keys: %w[paths responses]) do |node| + case node.key + when "paths" + operation[0] = node.annotation["method"] + operation[1] = node.annotation["current_path"] + when "responses" + operation[2] = node.annotation + end + end + covered_paths << operation + end + + def uncovered_paths + defined_paths - covered_paths + end + + def covered_percent + covered_paths.count * 100.0 / defined_paths.count + end + + def report + format.new(self).report + exit 1 if mode == :strict && uncovered_paths.any? + end + + private + + def find_defined_paths(schema) + Set.new.tap do |paths| + schema["paths"].each do |path, path_item| + resolved_path_item = (path_item.key?("$ref") ? path_item.resolve_ref(path_item["$ref"]) : path_item) + resolved_path_item.slice("get", "post", "put", "patch", "delete", "options", "head", "trace").each do |method, operation| + operation["responses"]&.each do |code, _| + paths << [method, path, code] + end + end + end + end + end + end +end diff --git a/lib/skooma/matchers/conform_request_schema.rb b/lib/skooma/matchers/conform_request_schema.rb index 5808535..26e9a28 100644 --- a/lib/skooma/matchers/conform_request_schema.rb +++ b/lib/skooma/matchers/conform_request_schema.rb @@ -5,13 +5,17 @@ module Skooma module Matchers class ConformRequestSchema - def initialize(schema, mapped_response) - @schema = schema + def initialize(skooma, mapped_response) + @skooma = skooma + @schema = skooma.schema @mapped_response = mapped_response end def matches?(*) @result = @schema.evaluate(@mapped_response) + + @skooma.coverage.track_request(@result) if @mapped_response["response"] + @result.valid? end diff --git a/lib/skooma/matchers/conform_response_schema.rb b/lib/skooma/matchers/conform_response_schema.rb index b6f89a1..b0e83b0 100644 --- a/lib/skooma/matchers/conform_response_schema.rb +++ b/lib/skooma/matchers/conform_response_schema.rb @@ -3,8 +3,8 @@ module Skooma module Matchers class ConformResponseSchema < ConformRequestSchema - def initialize(schema, mapped_response, expected) - super(schema, mapped_response) + def initialize(skooma, mapped_response, expected) + super(skooma, mapped_response) @expected = expected end diff --git a/lib/skooma/matchers/wrapper.rb b/lib/skooma/matchers/wrapper.rb index ffcbe0f..1065be2 100644 --- a/lib/skooma/matchers/wrapper.rb +++ b/lib/skooma/matchers/wrapper.rb @@ -38,30 +38,39 @@ def response_object raise "Response object not found" end + + def skooma_openapi_schema + skooma.schema + end end - def initialize(helper_methods_module, openapi_path, base_uri: "https://skoomarb.dev/", path_prefix: "") + def initialize(helper_methods_module, openapi_path, base_uri: "https://skoomarb.dev/", path_prefix: "", **params) super() registry = create_test_registry pathname = Pathname.new(openapi_path) - source_uri = "#{base_uri}#{path_prefix.delete_suffix("/")}" + source_uri = "#{base_uri}#{path_prefix.delete_suffix("/").delete_prefix("/")}" source_uri += "/" unless source_uri.end_with?("/") registry.add_source( source_uri, JSONSkooma::Sources::Local.new(pathname.dirname.to_s) ) - schema = registry.schema(URI.parse("#{source_uri}#{pathname.basename}"), schema_class: Skooma::Objects::OpenAPI) - schema.path_prefix = path_prefix + @schema = registry.schema(URI.parse("#{source_uri}#{pathname.basename}"), schema_class: Skooma::Objects::OpenAPI) + @schema.path_prefix = path_prefix + + @coverage = Coverage.new(@schema, mode: params[:coverage], format: params[:coverage_format]) include DefaultHelperMethods include helper_methods_module - define_method :skooma_openapi_schema do - schema + skooma_self = self + define_method :skooma do + skooma_self end end + attr_accessor :schema, :coverage + private def create_test_registry diff --git a/lib/skooma/minitest.rb b/lib/skooma/minitest.rb index fb4f803..01e7f29 100644 --- a/lib/skooma/minitest.rb +++ b/lib/skooma/minitest.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "minitest/unit" + module Skooma # Minitest helpers for OpenAPI schema validation # @example @@ -10,19 +12,19 @@ module Skooma class Minitest < Matchers::Wrapper module HelperMethods def assert_conform_schema(expected_status) - matcher = Matchers::ConformSchema.new(skooma_openapi_schema, mapped_response, expected_status) + matcher = Matchers::ConformSchema.new(skooma, mapped_response, expected_status) assert matcher.matches?, -> { matcher.failure_message } end def assert_conform_request_schema - matcher = Matchers::ConformRequestSchema.new(skooma_openapi_schema, mapped_response(with_response: false)) + matcher = Matchers::ConformRequestSchema.new(skooma, mapped_response(with_response: false)) assert matcher.matches?, -> { matcher.failure_message } end def assert_conform_response_schema(expected_status) - matcher = Matchers::ConformResponseSchema.new(skooma_openapi_schema, mapped_response(with_request: false), expected_status) + matcher = Matchers::ConformResponseSchema.new(skooma, mapped_response(with_request: false), expected_status) assert matcher.matches?, -> { matcher.failure_message } end @@ -36,6 +38,8 @@ def assert_is_valid_document(document) def initialize(openapi_path, **params) super(HelperMethods, openapi_path, **params) + + MiniTest::Unit.after_tests { coverage.report } end end end diff --git a/lib/skooma/objects/openapi/keywords/paths.rb b/lib/skooma/objects/openapi/keywords/paths.rb index 175019d..b20b917 100644 --- a/lib/skooma/objects/openapi/keywords/paths.rb +++ b/lib/skooma/objects/openapi/keywords/paths.rb @@ -28,6 +28,8 @@ def evaluate(instance, result) return result.failure("Path #{instance["path"]} not found in schema") unless path + result.annotate({"current_path" => path}) + result.call(instance, path) do |subresult| subresult.annotate({"path_attributes" => attributes}) path_schema.evaluate(instance, subresult) diff --git a/lib/skooma/objects/path_item/keywords/base_operation.rb b/lib/skooma/objects/path_item/keywords/base_operation.rb index c488d8d..ffa397a 100644 --- a/lib/skooma/objects/path_item/keywords/base_operation.rb +++ b/lib/skooma/objects/path_item/keywords/base_operation.rb @@ -13,12 +13,16 @@ def evaluate(instance, result) end json.evaluate(instance, result) - return result.success if result.passed? path_item_result = result.parent path_item_result = path_item_result.parent until path_item_result.key.start_with?("/") - path = path_item_result.annotation["path"] + paths_result = path_item_result.parent + paths_result.annotate(paths_result.annotation.merge("method" => key)) + + return result.success if result.passed? + + path = paths_result.annotation["current_path"] result.failure("Path #{path}/#{key} is invalid") end diff --git a/lib/skooma/objects/path_item/keywords/delete.rb b/lib/skooma/objects/path_item/keywords/delete.rb index 7e18633..56d8a73 100644 --- a/lib/skooma/objects/path_item/keywords/delete.rb +++ b/lib/skooma/objects/path_item/keywords/delete.rb @@ -5,7 +5,7 @@ module Objects class PathItem module Keywords class Delete < BaseOperation - self.key = "options" + self.key = "delete" self.depends_on = %w[parameters] self.value_schema = :schema self.schema_value_class = Objects::Operation diff --git a/lib/skooma/rspec.rb b/lib/skooma/rspec.rb index 7d11d7a..9a3abf4 100644 --- a/lib/skooma/rspec.rb +++ b/lib/skooma/rspec.rb @@ -10,15 +10,15 @@ module Skooma class RSpec < Matchers::Wrapper module HelperMethods def conform_schema(expected_status) - Matchers::ConformSchema.new(skooma_openapi_schema, mapped_response, expected_status) + Matchers::ConformSchema.new(skooma, mapped_response, expected_status) end def conform_response_schema(expected_status) - Matchers::ConformResponseSchema.new(skooma_openapi_schema, mapped_response(with_request: false), expected_status) + Matchers::ConformResponseSchema.new(skooma, mapped_response(with_request: false), expected_status) end def conform_request_schema - Matchers::ConformRequestSchema.new(skooma_openapi_schema, mapped_response(with_response: false)) + Matchers::ConformRequestSchema.new(skooma, mapped_response(with_response: false)) end def be_valid_document @@ -28,6 +28,13 @@ def be_valid_document def initialize(openapi_path, **params) super(HelperMethods, openapi_path, **params) + + skooma_self = self + ::RSpec.configure do |c| + c.after(:suite) do + at_exit { skooma_self.coverage.report } + end + end end end end