diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 22c01597..282ff322 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -57,28 +57,31 @@ def execute_in_parallel(items, num_processes, options) def run_tests_in_parallel(num_processes, options) test_results = nil + consolidated_results = {} - report_time_taken do + report_time_taken(consolidated_results) do groups = @runner.tests_in_groups(options[:files], num_processes, options) - groups.reject! &:empty? + groups.reject!(&:empty?) test_results = if options[:only_group] groups_to_run = options[:only_group].collect{|i| groups[i - 1]}.compact - report_number_of_tests(groups_to_run) + report_number_of_tests(groups_to_run, consolidated_results) execute_in_parallel(groups_to_run, groups_to_run.size, options) do |group| run_tests(group, groups_to_run.index(group), 1, options) end else - report_number_of_tests(groups) + report_number_of_tests(groups, consolidated_results) execute_in_parallel(groups, groups.size, options) do |group| run_tests(group, groups.index(group), num_processes, options) end end - report_results(test_results, options) + report_results(test_results, consolidated_results, options) end + write_consolidated_results(consolidated_results, options) + abort final_fail_message if any_test_failed?(test_results) end @@ -109,10 +112,12 @@ def lock(lockfile) end end - def report_results(test_results, options) + def report_results(test_results, consolidated_results, options) results = @runner.find_results(test_results.map { |result| result[:stdout] }*"") + @runner.summarize_results(results, consolidated_results) + puts "" - puts @runner.summarize_results(results) + puts consolidated_results[:summary][:message] report_failure_rerun_commmand(test_results, options) end @@ -132,12 +137,27 @@ def report_failure_rerun_commmand(test_results, options) end end - def report_number_of_tests(groups) + def report_number_of_tests(groups, consolidated_results) name = @runner.test_file_name num_processes = groups.size num_tests = groups.map(&:size).inject(0, :+) tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes) - puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" + message = "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process" + consolidated_results[:tests] = { + message: message, + tests: num_tests, + processes: num_processes, + tests_per_process: tests_per_process + } + puts message + end + + def write_consolidated_results(consolidated_results, options) + return if options[:consolidated_results].nil? || options[:consolidated_results].empty? + + File.write(options[:consolidated_results], consolidated_results.to_json) + + puts "Wrote consolidated results to #{options[:consolidated_results]}" end #exit with correct status code so rake parallel:test && echo 123 works @@ -216,6 +236,7 @@ def parse_options!(argv) opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time } opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true } opts.on("--verbose", "Print more output") { options[:verbose] = true } + opts.on("--consolidated-results [PATH]", "Location to write consolidated test results") { |path| options[:consolidated_results] = path } opts.on("-v", "--version", "Show Version") { puts ParallelTests::VERSION; exit } opts.on("-h", "--help", "Show this.") { puts opts; exit } end.parse!(argv) @@ -291,9 +312,14 @@ def execute_shell_command_in_parallel(command, num_processes, options) abort if results.any? { |r| r[:exit_status] != 0 } end - def report_time_taken + def report_time_taken(consolidated_results) seconds = ParallelTests.delta { yield }.to_i - puts "\nTook #{seconds} seconds#{detailed_duration(seconds)}" + message = "Took #{seconds} seconds#{detailed_duration(seconds)}" + consolidated_results[:time_taken] = { + seconds: seconds, + message: message + } + puts "\n#{message}" end def detailed_duration(seconds) diff --git a/lib/parallel_tests/cucumber/runner.rb b/lib/parallel_tests/cucumber/runner.rb index 917a0951..6ee592a4 100644 --- a/lib/parallel_tests/cucumber/runner.rb +++ b/lib/parallel_tests/cucumber/runner.rb @@ -15,7 +15,7 @@ def line_is_result?(line) super || line =~ SCENARIO_REGEX || line =~ SCENARIOS_RESULTS_BOUNDARY_REGEX end - def summarize_results(results) + def summarize_results(results, consolidated_results) output = [] scenario_groups = results.slice_before(SCENARIOS_RESULTS_BOUNDARY_REGEX).group_by(&:first) @@ -28,7 +28,11 @@ def summarize_results(results) output << super - output.join("\n\n") + consolidated_results[:summary] = { + message: output.join("\n\n") + } + + consolidated_results[:summary][:message] end def command_with_seed(cmd, seed) diff --git a/lib/parallel_tests/gherkin/runner.rb b/lib/parallel_tests/gherkin/runner.rb index 3429511f..f33c4ab3 100644 --- a/lib/parallel_tests/gherkin/runner.rb +++ b/lib/parallel_tests/gherkin/runner.rb @@ -43,10 +43,10 @@ def line_is_result?(line) # cucumber has 2 result lines per test run, that cannot be added # 1 scenario (1 failed) # 1 step (1 failed) - def summarize_results(results) + def summarize_results(results, consolidated_results) sort_order = %w[scenario step failed flaky undefined skipped pending passed] - %w[scenario step].map do |group| + message = %w[scenario step].map do |group| group_results = results.grep(/^\d+ #{group}/) next if group_results.empty? @@ -58,6 +58,12 @@ def summarize_results(results) end "#{sums[0]} (#{sums[1..-1].join(", ")})" end.compact.join("\n") + + consolidated_results[:summary] = { + message: message + } + + consolidated_results[:summary][:message] end def cucumber_opts(given) diff --git a/lib/parallel_tests/test/runner.rb b/lib/parallel_tests/test/runner.rb index 0c7dfc08..0d89a2db 100644 --- a/lib/parallel_tests/test/runner.rb +++ b/lib/parallel_tests/test/runner.rb @@ -114,9 +114,15 @@ def test_env_number(process_number, options={}) end end - def summarize_results(results) + def summarize_results(results, consolidated_results) sums = sum_up_results(results) - sums.sort.map{|word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ') + message = sums.sort.map{|word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ') + + consolidated_results[:summary] = sums.merge({ + message: message + }) + + consolidated_results[:summary][:message] end # remove old seed and add new seed diff --git a/spec/parallel_tests/cucumber/runner_spec.rb b/spec/parallel_tests/cucumber/runner_spec.rb index e9f6a32a..35e2fbd0 100644 --- a/spec/parallel_tests/cucumber/runner_spec.rb +++ b/spec/parallel_tests/cucumber/runner_spec.rb @@ -20,7 +20,7 @@ def call(*args) "Failing Scenarios:", "cucumber features/failure:3", "cucumber features/failure:4", "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6" ] - expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:5\ncucumber features/failure:6\n\n") + expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:5\ncucumber features/failure:6\n\n") end it "collates flaky scenarios separately" do @@ -30,7 +30,7 @@ def call(*args) "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6", "Flaky Scenarios:", "cucumber features/failure:7", "cucumber features/failure:8", ] - expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n") + expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n") end end end diff --git a/spec/parallel_tests/gherkin/runner_behaviour.rb b/spec/parallel_tests/gherkin/runner_behaviour.rb index 558edd44..74dbf3ec 100644 --- a/spec/parallel_tests/gherkin/runner_behaviour.rb +++ b/spec/parallel_tests/gherkin/runner_behaviour.rb @@ -179,7 +179,7 @@ def call(*args) "4 scenarios (4 passed)", "40 steps (40 passed)", "1 scenario (1 passed)", "1 step (1 passed)" ] - expect(call(results)).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)") + expect(call(results, {})).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)") end it "adds same results with plurals" do @@ -187,7 +187,7 @@ def call(*args) "1 scenario (1 passed)", "2 steps (2 passed)", "2 scenarios (2 passed)", "7 steps (7 passed)" ] - expect(call(results)).to eq("3 scenarios (3 passed)\n9 steps (9 passed)") + expect(call(results, {})).to eq("3 scenarios (3 passed)\n9 steps (9 passed)") end it "adds non-similar results" do @@ -195,11 +195,11 @@ def call(*args) "1 scenario (1 passed)", "1 step (1 passed)", "2 scenarios (1 failed, 1 pending)", "2 steps (1 failed, 1 pending)" ] - expect(call(results)).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)") + expect(call(results, {})).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)") end it "does not pluralize 1" do - expect(call(["1 scenario (1 passed)", "1 step (1 passed)"])).to eq("1 scenario (1 passed)\n1 step (1 passed)") + expect(call(["1 scenario (1 passed)", "1 step (1 passed)"], {})).to eq("1 scenario (1 passed)\n1 step (1 passed)") end end diff --git a/spec/parallel_tests/test/runner_spec.rb b/spec/parallel_tests/test/runner_spec.rb index 6987f25f..b7e839b6 100644 --- a/spec/parallel_tests/test/runner_spec.rb +++ b/spec/parallel_tests/test/runner_spec.rb @@ -316,23 +316,23 @@ def call(*args) end it "adds results" do - expect(call(['1 foo 3 bar','2 foo 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo 3 bar','2 foo 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds results with braces" do - expect(call(['1 foo(s) 3 bar(s)','2 foo 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo(s) 3 bar(s)','2 foo 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds same results with plurals" do - expect(call(['1 foo 3 bar','2 foos 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo 3 bar','2 foos 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds non-similar results" do - expect(call(['1 xxx 2 yyy','1 xxx 2 zzz'])).to eq('2 xxxs, 2 yyys, 2 zzzs') + expect(call(['1 xxx 2 yyy','1 xxx 2 zzz'], {})).to eq('2 xxxs, 2 yyys, 2 zzzs') end it "does not pluralize 1" do - expect(call(['1 xxx 2 yyy'])).to eq('1 xxx, 2 yyys') + expect(call(['1 xxx 2 yyy'], {})).to eq('1 xxx, 2 yyys') end end