From aadcc549ec7004b8dc88ebabc2caf1128058c32a Mon Sep 17 00:00:00 2001 From: Tina Wuest Date: Wed, 13 Mar 2024 02:12:04 +0200 Subject: [PATCH] Add result persistence --- .rubocop.yml | 10 +++ lib/minitest/proptest.rb | 128 ++++++++++++++++++++++++++++++ lib/minitest/proptest/property.rb | 43 +++++++++- lib/minitest/proptest_plugin.rb | 46 +++++++++-- 4 files changed, 218 insertions(+), 9 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index a457709..ee41aa1 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,6 +1,10 @@ AllCops: NewCops: enable +Layout/LineLength: + Exclude: + - 'lib/minitest/proptest_plugin.rb' + Layout/SpaceInsideParens: Enabled: false @@ -50,9 +54,15 @@ Style/Lambda: Style/MultilineBlockChain: Enabled: false +Layout/MultilineHashBraceLayout: + Enabled: false + Style/NumericPredicate: Enabled: false +Style/OptionalBooleanParameter: + Enabled: false + Style/RedundantSelf: Enabled: false diff --git a/lib/minitest/proptest.rb b/lib/minitest/proptest.rb index 546300a..a3dff34 100644 --- a/lib/minitest/proptest.rb +++ b/lib/minitest/proptest.rb @@ -5,18 +5,146 @@ require 'minitest/proptest/property' require 'minitest/proptest/status' require 'minitest/proptest/version' +require 'yaml' module Minitest + class ResultsDatabase < Minitest::AbstractReporter + def initialize(pathname) + super() + + results = if File.file?(pathname) + YAML.load_file(pathname) + else + {} + end + self.class.instance_variable_set(:@_results, results) unless self.class.instance_variable_defined?(:@_results) + end + + def report + return unless Proptest.use_db? + + File.write(Proptest.result_db, self.class.instance_variable_get(:@_results).to_yaml) + end + + def lookup(file, classname, methodname) + self.class.instance_variable_get(:@_results) + .dig(file, classname, methodname) + .to_a + end + + def record_failure(file, classname, methodname, generated) + return unless Proptest.use_db? + + results = self.class.instance_variable_get(:@_results) + results[file] ||= {} + results[file][classname] ||= {} + results[file][classname][methodname] = generated + end + + def strike_failure(file, classname, methodname) + return unless Proptest.use_db? + + results = self.class.instance_variable_get(:@_results) + return unless results.key?(file) + + return unless results[file].key?(classname) + + results[file][classname].delete(methodname) + results[file].delete(classname) if results[file][classname].empty? + results.delete(file) if results[file].empty? + end + end + module Proptest DEFAULT_RANDOM = Random.method(:new) DEFAULT_MAX_SUCCESS = 100 DEFAULT_MAX_DISCARD_RATIO = 10 DEFAULT_MAX_SIZE = 0x100 DEFAULT_MAX_SHRINKS = (((1 << (1.size * 8)) - 1) / 2) + DEFAULT_DB_LOCATION = File.join(Dir.pwd, '.proptest_failures.yml') + + self.instance_variable_set(:@_random, DEFAULT_RANDOM) + self.instance_variable_set(:@_max_success, DEFAULT_MAX_SUCCESS) + self.instance_variable_set(:@_max_discard_ratio, DEFAULT_MAX_DISCARD_RATIO) + self.instance_variable_set(:@_max_size, DEFAULT_MAX_SIZE) + self.instance_variable_set(:@_max_shrinks, DEFAULT_MAX_SHRINKS) + self.instance_variable_set(:@_result_db, DEFAULT_DB_LOCATION) + self.instance_variable_set(:@_use_db, false) def self.set_seed(seed) self.instance_variable_set(:@_random_seed, seed) end + + def self.max_success=(success) + self.instance_variable_set(:@_max_success, success) + end + + def self.max_discard_ratio=(discards) + self.instance_variable_set(:@_max_discard_ratio, discards) + end + + def self.max_size=(size) + self.instance_variable_set(:@_max_size, size) + end + + def self.max_shrinks=(shrinks) + self.instance_variable_set(:@_max_shrinks, shrinks) + end + + def self.result_db=(location) + self.instance_variable_set(:@_result_db, File.expand_path(location)) + end + + def self.use_db!(use = true) + self.instance_variable_set(:@_use_db, use) + end + + def self.seed + self.instance_variable_get(:@_random_seed) + end + + def self.max_success + self.instance_variable_get(:@_max_success) + end + + def self.max_discard_ratio + self.instance_variable_get(:@_max_discard_ratio) + end + + def self.max_size + self.instance_variable_get(:@_max_size) + end + + def self.max_shrinks + self.instance_variable_get(:@_max_shrinks) + end + + def self.result_db + self.instance_variable_get(:@_result_db) + end + + def self.use_db? + self.instance_variable_get(:@_use_db) + end + + def self.record_failure(file, classname, methodname, generated) + self.instance_variable_get(:@_results) + .record_failure(file, classname, methodname, generated) + end + + def self.strike_failure(file, classname, methodname) + self.instance_variable_get(:@_results) + .strike_failure(file, classname, methodname) + end + + def self.reporter + return self.instance_variable_get(:@_results) if self.instance_variable_defined?(:@_results) + + reporter = Minitest::ResultsDatabase.new(result_db) + self.instance_variable_set(:@_results, reporter) + + reporter + end end end diff --git a/lib/minitest/proptest/property.rb b/lib/minitest/proptest/property.rb index 2085ad0..56404f4 100644 --- a/lib/minitest/proptest/property.rb +++ b/lib/minitest/proptest/property.rb @@ -22,7 +22,10 @@ def initialize( max_size: 0x100, # Maximum number of shrink attempts (default of half of max unsigned int # on the system architecture adopted from QuickCheck - max_shrinks: 0x7fffffffffffffff + max_shrinks: 0x7fffffffffffffff, + # Previously discovered counter-example. If this exists, it should be + # run before any test cases are generated. + previous_failure: [] ) @test_proc = test_proc @random = random.call @@ -39,9 +42,11 @@ def initialize( @valid_test_cases = 0 @generated = [] @arbitrary = nil + @previous_failure = previous_failure.to_a end def run! + rerun! iterate! shrink! end @@ -109,6 +114,38 @@ def iterate! @exception = e end + def rerun! + return if @previous_failure.empty? + + old_generator = @generator + old_random = @random + old_arbitrary = @arbitrary + + index = -1 + @arbitrary = ->(*classes) do + index += 1 + raise IndexError if index >= @previous_failure.length + + a = @generator.for(*classes) + a = a.force(@previous_failure[index]) + @generated << a + @previous_failure[index] + end + + @generator = ::Minitest::Proptest::Gen.new(@random) + if instance_eval(&@test_proc) + @generated = [] + else + @result = @generated + @status = Status.interesting + end + + # Clean up after we're done + @generator = old_generator + @random = old_random + @arbitrary = old_arbitrary + end + def shrink! return if @result.nil? @@ -144,8 +181,8 @@ def shrink! if to_test[run[:run]].map(&:first).reduce(&:+) < best_score unless instance_eval(&@test_proc) best_generated = @generated - # Because we pre-sorted our shrink candidates, the first hit is - # necessarily the best scoring + # The first hit is guaranteed to be the best scoring due to the + # shrink candidates are pre-sorted. break end end diff --git a/lib/minitest/proptest_plugin.rb b/lib/minitest/proptest_plugin.rb index 4e21dfe..fe02a5d 100644 --- a/lib/minitest/proptest_plugin.rb +++ b/lib/minitest/proptest_plugin.rb @@ -6,6 +6,7 @@ require 'minitest/proptest/property' require 'minitest/proptest/status' require 'minitest/proptest/version' +require 'yaml' module Minitest def self.plugin_proptest_init(options) @@ -20,10 +21,32 @@ def self.plugin_proptest_init(options) end end + self.reporter << Proptest.reporter + Proptest.set_seed(options[:seed]) if options.key?(:seed) end - def self.plugin_proptest_options(opts, options); end + def self.plugin_proptest_options(opts, _options) + opts.on('--max-success', Integer, "Maximum number of successful cases to verify for each property (Default: #{Minitest::Proptest::DEFAULT_MAX_SUCCESS})") do |max_success| + Proptest.max_success = max_success + end + opts.on('--max-discard-ratio', Integer, "Maximum ratio of successful cases versus discarded cases per property (Default: #{Minitest::Proptest::DEFAULT_MAX_DISCARD_RATIO}:1)") do |max_success| + Proptest.max_success = max_success + end + opts.on('--max-size', Integer, "Maximum amount of entropy a single case may use in bytes (Default: #{Minitest::Proptest::DEFAULT_MAX_SIZE} bytes)") do |max_size| + Proptest.max_size = max_size + end + opts.on('--max-shrinks', Integer, "Maximum number of shrink iterations a single failure reduction may use (Default: #{Minitest::Proptest::DEFAULT_MAX_SHRINKS})") do |max_shrinks| + Proptest.max_shrinks = max_shrinks + end + opts.on('--results-db', String, "Location of the file to persist most recent failure cases. Implies --use-db. (Default: #{Minitest::Proptest::DEFAULT_DB_LOCATION})") do |db_path| + Proptest.result_db = db_path + Proptest.use_db! + end + opts.on('--use-db', 'Persist previous failures in a database and use them before generating new values. Helps prevent flaky builds. (Default: false)') do + Proptest.use_db! + end + end module Assertions def property(&f) @@ -36,17 +59,28 @@ def property(&f) Proptest::DEFAULT_RANDOM end + file, methodname = caller.first.split(/:\d+:in +/) + classname = self.class.name + methodname.gsub!(/(?:^`|'$)/, '') + prop = Minitest::Proptest::Property.new( f, random: random_thunk, - max_success: Proptest::DEFAULT_MAX_SUCCESS, - max_discard_ratio: Proptest::DEFAULT_MAX_DISCARD_RATIO, - max_size: Proptest::DEFAULT_MAX_SIZE, - max_shrinks: Proptest::DEFAULT_MAX_SHRINKS + max_success: Proptest.max_success, + max_discard_ratio: Proptest.max_discard_ratio, + max_size: Proptest.max_size, + max_shrinks: Proptest.max_shrinks, + previous_failure: Proptest.reporter.lookup(file, classname, methodname) ) prop.run! - raise Minitest::Assertion, prop.explain unless prop.status.valid? && !prop.trivial + if prop.status.valid? && !prop.trivial + Proptest.strike_failure(file, classname, methodname) + else + Proptest.record_failure(file, classname, methodname, prop.result.map(&:value)) + + raise Minitest::Assertion, prop.explain + end end end end