Feature/support profile

.gitignore

@@ -10,7 +10,6 @@
 # rspec failure tracking
 .rspec_status
 
-
 # Created by https://www.toptal.com/developers/gitignore/api/macos,vim
 # Edit at https://www.toptal.com/developers/gitignore?templates=macos,vim
 
@@ -23,7 +22,6 @@
 # Icon must end with two \r
 Icon
 
-
 # Thumbnails
 ._*
 
@@ -46,7 +44,7 @@ Temporary Items
 ### Vim ###
 # Swap
 [._]*.s[a-v][a-z]
-!*.svg  # comment out if you don't need vector files
+!*.svg # comment out if you don't need vector files
 [._]*.sw[a-p]
 [._]s[a-rt-v][a-z]
 [._]ss[a-gi-z]
@@ -66,3 +64,4 @@ tags
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,vim
 
+.ruby-version

fixtures/rspec/6_tests_spec.rb

@@ -0,0 +1,26 @@
+
+RSpec.describe "Failing example group" do
+  context "When more than 10 tests" do  
+    context "when it has 6 tests" do
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+    end
+  end
+end
+

fixtures/rspec/another_6_tests_spec.rb

@@ -0,0 +1,26 @@
+
+RSpec.describe "Failing example group" do
+  context "When more than 10 tests" do  
+    context "when it has another 6 tests" do
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+      it "has a version number" do
+        expect(TurboTests::VERSION).not_to be nil
+      end
+    end 
+  end
+end
+

lib/turbo_tests/cli.rb

@@ -17,6 +17,7 @@ def run
       verbose = false
       fail_fast = nil
       seed = nil
+      profile = nil
 
       OptionParser.new { |opts|
         opts.banner = <<~BANNER
@@ -81,6 +82,15 @@ def run
         opts.on("--seed SEED", "Seed for rspec") do |s|
           seed = s
         end
+
+        opts.on("--profile N", "Slowest N tests") do |n|
+          n = begin
+            Integer(n)
+          rescue
+            raise ArgumentError, "Invalid argument for --profile. Must be an integer."
+          end
+          profile = n
+        end
       }.parse!(@argv)
 
       requires.each { |f| require(f) }
@@ -99,14 +109,15 @@ def run
       end
 
       exitstatus = TurboTests::Runner.run(
-        formatters: formatters,
-        tags: tags,
+        formatters:,
+        tags:,
         files: @argv.empty? ? ["spec"] : @argv,
-        runtime_log: runtime_log,
-        verbose: verbose,
-        fail_fast: fail_fast,
-        count: count,
-        seed: seed
+        runtime_log:,
+        verbose:,
+        fail_fast:,
+        count:,
+        seed:,
+        profile:
       )
 
       # From https://github.com/serpapi/turbo_tests/pull/20/

lib/turbo_tests/json_rows_formatter.rb

@@ -25,6 +25,7 @@ class JsonRowsFormatter
       :start,
       :close,
       :example_failed,
+      :dump_profile,
       :example_passed,
       :example_pending,
       :example_group_started,
@@ -46,6 +47,13 @@ def start(notification)
       )
     end
 
+    def dump_profile(notification)
+      output_row(
+        type: :dump_profile,
+        dump_profile: dump_profile_to_json(notification)
+      )
+    end
+
     def example_group_started(notification)
       output_row(
         type: :group_started,
@@ -114,13 +122,16 @@ def exception_to_json(exception)
       end
     end
 
+    ExampleExecutionResult = Struct.new(:example_skipped?, :pending_message, :status, :pending_fixed?, :exception, :run_time)
+
     def execution_result_to_json(result)
       {
         example_skipped?: result.example_skipped?,
         pending_message: result.pending_message,
         status: result.status,
         pending_fixed?: result.pending_fixed?,
-        exception: exception_to_json(result.exception || result.pending_exception)
+        exception: exception_to_json(result.exception || result.pending_exception),
+        run_time: result.run_time
       }
     end
 
@@ -130,6 +141,8 @@ def stack_frame_to_json(frame)
         inclusion_location: frame.inclusion_location
       }
     end
+
+    Example = Struct.new(:execution_result, :location, :description, :full_description, :metadata, :location_rerun_argument)
 
     def example_to_json(example)
       {
@@ -155,6 +168,15 @@ def load_summary_to_json(notification)
       }
     end
 
+    def dump_profile_to_json(notification)
+      {
+        duration: notification.duration,
+        examples: notification.examples.map { |example| example_to_json(example) },
+      }
+    end
+
+    Group = Struct.new(:description, :location, :total_time, :count, :average)
+
     def group_to_json(notification)
       {
         group: {

lib/turbo_tests/reporter.rb

@@ -4,8 +4,8 @@ module TurboTests
   class Reporter
     attr_writer :load_time
 
-    def self.from_config(formatter_config, start_time, seed, seed_used)
-      reporter = new(start_time, seed, seed_used)
+    def self.from_config(formatter_config, start_time, seed, seed_used, profile)
+      reporter = new(start_time, seed, seed_used, profile)
 
       formatter_config.each do |config|
         name, outputs = config.values_at(:name, :outputs)
@@ -23,11 +23,15 @@ def self.from_config(formatter_config, start_time, seed, seed_used)
     attr_reader :pending_examples
     attr_reader :failed_examples
 
-    def initialize(start_time, seed, seed_used)
+    def initialize(start_time, seed, seed_used, profile)
       @formatters = []
       @pending_examples = []
       @failed_examples = []
+      @profile = profile
       @all_examples = []
+      @all_profile_examples = []
+      @all_profile_groups = {}
+      @profile_time = 0
       @messages = []
       @start_time = start_time
       @seed = seed
@@ -44,6 +48,8 @@ def add(name, outputs)
             RSpec::Core::Formatters::ProgressFormatter
           when "d", "documentation"
             RSpec::Core::Formatters::DocumentationFormatter
+          when "profile"
+            RSpec::Core::Formatters::ProfileFormatter
           else
             Kernel.const_get(name)
           end
@@ -112,6 +118,19 @@ def example_failed(example)
       @failed_examples << example
     end
 
+    def dump_profile(profile)
+      group = JsonRowsFormatter::Group.new(
+        # TODO: this isnt correct, but we cant seem to access the existing group info as its private 
+        location:  profile[:examples].map { |e| e[:location].split(":").first }.uniq.join(", "),
+        description: "Group #{@all_profile_groups.keys.count + 1}",
+        count: profile[:examples].count,
+        total_time: profile[:examples].sum { |e| e[:execution_result][:run_time] },
+      )
+      @all_profile_groups[group] = group
+      @all_profile_examples += profile[:examples]
+      @profile_time += profile[:duration]
+    end
+
     def message(message)
       delegate_to_formatters(:message, RSpec::Core::Notifications::MessageNotification.new(message))
       @messages << message
@@ -146,6 +165,18 @@ def finish
           @load_time,
           @errors_outside_of_examples_count
         ))
+      delegate_to_formatters(:dump_profile,
+        RSpec::Core::Notifications::ProfileNotification.new(
+          @profile_time,
+          @all_profile_examples.map do |e| 
+            JsonRowsFormatter::Example.new(
+              **e, 
+              execution_result: JsonRowsFormatter::ExampleExecutionResult.new(**e[:execution_result])
+            ) 
+          end,
+          @profile,
+          @all_profile_groups
+        )) if @all_profile_examples.any?
       delegate_to_formatters(:seed,
         RSpec::Core::Notifications::SeedNotification.new(
           @seed,

lib/turbo_tests/runner.rb

@@ -20,24 +20,28 @@ def self.run(opts = {})
       fail_fast = opts.fetch(:fail_fast, nil)
       count = opts.fetch(:count, nil)
       seed = opts.fetch(:seed)
+      profile = opts.fetch(:profile)
       seed_used = !seed.nil?
 
+      formatters << { name: "profile", outputs: ["-"] } unless profile.nil?
+
       if verbose
         warn "VERBOSE"
       end
 
-      reporter = Reporter.from_config(formatters, start_time, seed, seed_used)
+      reporter = Reporter.from_config(formatters, start_time, seed, seed_used, profile)
 
       new(
-        reporter: reporter,
-        files: files,
-        tags: tags,
-        runtime_log: runtime_log,
-        verbose: verbose,
-        fail_fast: fail_fast,
-        count: count,
-        seed: seed,
-        seed_used: seed_used,
+        reporter:,
+        files:,
+        tags:,
+        runtime_log:,
+        verbose:,
+        fail_fast:,
+        count:,
+        seed:,
+        seed_used:,
+        profile:
       ).run
     end
 
@@ -50,6 +54,7 @@ def initialize(opts)
       @fail_fast = opts[:fail_fast]
       @count = opts[:count]
       @seed = opts[:seed]
+      @profile = opts[:profile]
       @seed_used = opts[:seed_used]
 
       @load_time = 0
@@ -164,10 +169,19 @@ def start_subprocess(env, extra_args, tests, process_id, record_runtime:)
           []
         end
 
+        profile_option = if @profile
+          [
+            "--profile", @profile.to_s,
+          ]
+        else
+          []
+        end
+
         command = [
           *command_name,
           *extra_args,
           *seed_option,
+          *profile_option,
           "--format", "TurboTests::JsonRowsFormatter",
           *record_runtime_options,
           *tests,
@@ -231,10 +245,12 @@ def start_copy_thread(src, dst)
 
     def handle_messages
       exited = 0
-
+      
       loop do
         message = @messages.pop
         case message[:type]
+        when "dump_profile"
+          @reporter.dump_profile(message[:dump_profile])
         when "example_passed"
           example = FakeExample.from_obj(message[:example])
           @reporter.example_passed(example)