Add result persistence

.rubocop.yml

@@ -1,6 +1,10 @@
 AllCops:
   NewCops: enable
 
+Layout/LineLength:
+  Exclude:
+    - 'lib/minitest/proptest_plugin.rb'
+
 Layout/SpaceInsideParens:
   Enabled: false
 
@@ -50,9 +54,15 @@ Style/Lambda:
 Style/MultilineBlockChain:
   Enabled: false
 
+Layout/MultilineHashBraceLayout:
+  Enabled: false
+
 Style/NumericPredicate:
   Enabled: false
 
+Style/OptionalBooleanParameter:
+  Enabled: false
+
 Style/RedundantSelf:
   Enabled: false
 

lib/minitest/proptest.rb

@@ -5,18 +5,146 @@
 require 'minitest/proptest/property'
 require 'minitest/proptest/status'
 require 'minitest/proptest/version'
+require 'yaml'
 
 module Minitest
+  class ResultsDatabase < Minitest::AbstractReporter
+    def initialize(pathname)
+      super()
+
+      results = if File.file?(pathname)
+                  YAML.load_file(pathname)
+                else
+                  {}
+                end
+      self.class.instance_variable_set(:@_results, results) unless self.class.instance_variable_defined?(:@_results)
+    end
+
+    def report
+      return unless Proptest.use_db?
+
+      File.write(Proptest.result_db, self.class.instance_variable_get(:@_results).to_yaml)
+    end
+
+    def lookup(file, classname, methodname)
+      self.class.instance_variable_get(:@_results)
+          .dig(file, classname, methodname)
+          .to_a
+    end
+
+    def record_failure(file, classname, methodname, generated)
+      return unless Proptest.use_db?
+
+      results = self.class.instance_variable_get(:@_results)
+      results[file] ||= {}
+      results[file][classname] ||= {}
+      results[file][classname][methodname] = generated
+    end
+
+    def strike_failure(file, classname, methodname)
+      return unless Proptest.use_db?
+
+      results = self.class.instance_variable_get(:@_results)
+      return unless results.key?(file)
+
+      return unless results[file].key?(classname)
+
+      results[file][classname].delete(methodname)
+      results[file].delete(classname) if results[file][classname].empty?
+      results.delete(file) if results[file].empty?
+    end
+  end
+
   module Proptest
     DEFAULT_RANDOM = Random.method(:new)
     DEFAULT_MAX_SUCCESS = 100
     DEFAULT_MAX_DISCARD_RATIO = 10
     DEFAULT_MAX_SIZE = 0x100
     DEFAULT_MAX_SHRINKS = (((1 << (1.size * 8)) - 1) / 2)
+    DEFAULT_DB_LOCATION = File.join(Dir.pwd, '.proptest_failures.yml')
+
+    self.instance_variable_set(:@_random, DEFAULT_RANDOM)
+    self.instance_variable_set(:@_max_success, DEFAULT_MAX_SUCCESS)
+    self.instance_variable_set(:@_max_discard_ratio, DEFAULT_MAX_DISCARD_RATIO)
+    self.instance_variable_set(:@_max_size, DEFAULT_MAX_SIZE)
+    self.instance_variable_set(:@_max_shrinks, DEFAULT_MAX_SHRINKS)
+    self.instance_variable_set(:@_result_db, DEFAULT_DB_LOCATION)
+    self.instance_variable_set(:@_use_db, false)
 
     def self.set_seed(seed)
       self.instance_variable_set(:@_random_seed, seed)
     end
+
+    def self.max_success=(success)
+      self.instance_variable_set(:@_max_success, success)
+    end
+
+    def self.max_discard_ratio=(discards)
+      self.instance_variable_set(:@_max_discard_ratio, discards)
+    end
+
+    def self.max_size=(size)
+      self.instance_variable_set(:@_max_size, size)
+    end
+
+    def self.max_shrinks=(shrinks)
+      self.instance_variable_set(:@_max_shrinks, shrinks)
+    end
+
+    def self.result_db=(location)
+      self.instance_variable_set(:@_result_db, File.expand_path(location))
+    end
+
+    def self.use_db!(use = true)
+      self.instance_variable_set(:@_use_db, use)
+    end
+
+    def self.seed
+      self.instance_variable_get(:@_random_seed)
+    end
+
+    def self.max_success
+      self.instance_variable_get(:@_max_success)
+    end
+
+    def self.max_discard_ratio
+      self.instance_variable_get(:@_max_discard_ratio)
+    end
+
+    def self.max_size
+      self.instance_variable_get(:@_max_size)
+    end
+
+    def self.max_shrinks
+      self.instance_variable_get(:@_max_shrinks)
+    end
+
+    def self.result_db
+      self.instance_variable_get(:@_result_db)
+    end
+
+    def self.use_db?
+      self.instance_variable_get(:@_use_db)
+    end
+
+    def self.record_failure(file, classname, methodname, generated)
+      self.instance_variable_get(:@_results)
+          .record_failure(file, classname, methodname, generated)
+    end
+
+    def self.strike_failure(file, classname, methodname)
+      self.instance_variable_get(:@_results)
+          .strike_failure(file, classname, methodname)
+    end
+
+    def self.reporter
+      return self.instance_variable_get(:@_results) if self.instance_variable_defined?(:@_results)
+
+      reporter = Minitest::ResultsDatabase.new(result_db)
+      self.instance_variable_set(:@_results, reporter)
+
+      reporter
+    end
   end
 end
 

lib/minitest/proptest/property.rb

@@ -22,7 +22,10 @@ def initialize(
         max_size: 0x100,
         # Maximum number of shrink attempts (default of half of max unsigned int
         # on the system architecture adopted from QuickCheck
-        max_shrinks: 0x7fffffffffffffff
+        max_shrinks: 0x7fffffffffffffff,
+        # Previously discovered counter-example.  If this exists, it should be
+        # run before any test cases are generated.
+        previous_failure: []
       )
         @test_proc         = test_proc
         @random            = random.call
@@ -39,9 +42,11 @@ def initialize(
         @valid_test_cases  = 0
         @generated         = []
         @arbitrary         = nil
+        @previous_failure  = previous_failure.to_a
       end
 
       def run!
+        rerun!
         iterate!
         shrink!
       end
@@ -109,6 +114,38 @@ def iterate!
         @exception = e
       end
 
+      def rerun!
+        return if @previous_failure.empty?
+
+        old_generator  = @generator
+        old_random     = @random
+        old_arbitrary  = @arbitrary
+
+        index = -1
+        @arbitrary = ->(*classes) do
+          index += 1
+          raise IndexError if index >= @previous_failure.length
+
+          a = @generator.for(*classes)
+          a = a.force(@previous_failure[index])
+          @generated << a
+          @previous_failure[index]
+        end
+
+        @generator = ::Minitest::Proptest::Gen.new(@random)
+        if instance_eval(&@test_proc)
+          @generated = []
+        else
+          @result = @generated
+          @status = Status.interesting
+        end
+
+        # Clean up after we're done
+        @generator = old_generator
+        @random    = old_random
+        @arbitrary = old_arbitrary
+      end
+
       def shrink!
         return if @result.nil?
 
@@ -144,8 +181,8 @@ def shrink!
           if to_test[run[:run]].map(&:first).reduce(&:+) < best_score
             unless instance_eval(&@test_proc)
               best_generated = @generated
-              # Because we pre-sorted our shrink candidates, the first hit is
-              # necessarily the best scoring
+              # The first hit is guaranteed to be the best scoring due to the
+              # shrink candidates are pre-sorted.
               break
             end
           end

lib/minitest/proptest_plugin.rb

@@ -6,6 +6,7 @@
 require 'minitest/proptest/property'
 require 'minitest/proptest/status'
 require 'minitest/proptest/version'
+require 'yaml'
 
 module Minitest
   def self.plugin_proptest_init(options)
@@ -20,10 +21,32 @@ def self.plugin_proptest_init(options)
       end
     end
 
+    self.reporter << Proptest.reporter
+
     Proptest.set_seed(options[:seed]) if options.key?(:seed)
   end
 
-  def self.plugin_proptest_options(opts, options); end
+  def self.plugin_proptest_options(opts, _options)
+    opts.on('--max-success', Integer, "Maximum number of successful cases to verify for each property (Default: #{Minitest::Proptest::DEFAULT_MAX_SUCCESS})") do |max_success|
+      Proptest.max_success = max_success
+    end
+    opts.on('--max-discard-ratio', Integer, "Maximum ratio of successful cases versus discarded cases per property (Default: #{Minitest::Proptest::DEFAULT_MAX_DISCARD_RATIO}:1)") do |max_success|
+      Proptest.max_success = max_success
+    end
+    opts.on('--max-size', Integer, "Maximum amount of entropy a single case may use in bytes (Default: #{Minitest::Proptest::DEFAULT_MAX_SIZE} bytes)") do |max_size|
+      Proptest.max_size = max_size
+    end
+    opts.on('--max-shrinks', Integer, "Maximum number of shrink iterations a single failure reduction may use (Default: #{Minitest::Proptest::DEFAULT_MAX_SHRINKS})") do |max_shrinks|
+      Proptest.max_shrinks = max_shrinks
+    end
+    opts.on('--results-db', String, "Location of the file to persist most recent failure cases.  Implies --use-db.  (Default: #{Minitest::Proptest::DEFAULT_DB_LOCATION})") do |db_path|
+      Proptest.result_db = db_path
+      Proptest.use_db!
+    end
+    opts.on('--use-db', 'Persist previous failures in a database and use them before generating new values.  Helps prevent flaky builds.  (Default: false)') do
+      Proptest.use_db!
+    end
+  end
 
   module Assertions
     def property(&f)
@@ -36,17 +59,28 @@ def property(&f)
                        Proptest::DEFAULT_RANDOM
                      end
 
+      file, methodname = caller.first.split(/:\d+:in +/)
+      classname = self.class.name
+      methodname.gsub!(/(?:^`|'$)/, '')
+
       prop = Minitest::Proptest::Property.new(
         f,
         random: random_thunk,
-        max_success: Proptest::DEFAULT_MAX_SUCCESS,
-        max_discard_ratio: Proptest::DEFAULT_MAX_DISCARD_RATIO,
-        max_size: Proptest::DEFAULT_MAX_SIZE,
-        max_shrinks: Proptest::DEFAULT_MAX_SHRINKS
+        max_success: Proptest.max_success,
+        max_discard_ratio: Proptest.max_discard_ratio,
+        max_size: Proptest.max_size,
+        max_shrinks: Proptest.max_shrinks,
+        previous_failure: Proptest.reporter.lookup(file, classname, methodname)
       )
       prop.run!
 
-      raise Minitest::Assertion, prop.explain unless prop.status.valid? && !prop.trivial
+      if prop.status.valid? && !prop.trivial
+        Proptest.strike_failure(file, classname, methodname)
+      else
+        Proptest.record_failure(file, classname, methodname, prop.result.map(&:value))
+
+        raise Minitest::Assertion, prop.explain
+      end
     end
   end
 end