diff --git a/Makefile b/Makefile index 93edc562..768199a3 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,9 @@ bin/crytic: bin: build mkdir -p $(SHARD_BIN) cp ./bin/crytic $(SHARD_BIN) + +test-unit: + docker run --rm -it -v "$(shell pwd):/src" -w /src crystallang/crystal:0.32.0 /bin/sh -c "./bin/test-unit" + +test: + docker run --rm -it -v "$(shell pwd):/src" -w /src crystallang/crystal:0.32.0 /bin/sh -c "./bin/test" diff --git a/spec/fake_reporter.cr b/spec/fake_reporter.cr deleted file mode 100644 index d9485978..00000000 --- a/spec/fake_reporter.cr +++ /dev/null @@ -1,30 +0,0 @@ -require "../src/crytic/reporter/reporter" - -class FakeReporter < Crytic::Reporter::Reporter - getter events - @events = [] of String - - def report_original_result(original_result) - @events << "report_original_result" - end - - def report_mutations(mutations) - @events << "report_mutations" - end - - def report_neutral_result(result) - @events << "report_neutral_result" - end - - def report_result(result) - @events << "report_result" - end - - def report_summary(results) - @events << "report_summary" - end - - def report_msi(results) - @events << "report_msi" - end -end diff --git a/spec/mutation/no_mutation_spec.cr b/spec/mutation/no_mutation_spec.cr index 49d560e2..22a52b71 100644 --- a/spec/mutation/no_mutation_spec.cr +++ b/spec/mutation/no_mutation_spec.cr @@ -6,18 +6,18 @@ module Crytic::Mutation describe "#run" do it "runs crystal spec with a single spec file" do fake = FakeProcessRunner.new - mutation = NoMutation.with(["./single/test_spec.cr"], fake) + mutation = NoMutation.with(["./single/test_spec.cr"]) - mutation.run + mutation.run(side_effects(process_runner: fake)) fake.cmd_with_args.last.should eq "crystal spec ./single/test_spec.cr" end it "runs crystal spec with multiple spec files" do fake = FakeProcessRunner.new - mutation = NoMutation.with(["./a/b_spec.cr", "./a/c_spec.cr"], fake) + mutation = NoMutation.with(["./a/b_spec.cr", "./a/c_spec.cr"]) - mutation.run + mutation.run(side_effects(process_runner: fake)) fake.cmd_with_args.last.should eq "crystal spec ./a/b_spec.cr ./a/c_spec.cr" end diff --git a/spec/runner/sequential_spec.cr b/spec/runner/sequential_spec.cr index 263c679f..9c7ba9a0 100644 --- a/spec/runner/sequential_spec.cr +++ b/spec/runner/sequential_spec.cr @@ -9,73 +9,71 @@ module Crytic::Runner describe Sequential do describe "#run" do - it "takes a list of subjects" do - reporter = FakeReporter.new - runner = Sequential.new( - threshold: 100.0, - generator: FakeGenerator.new, - reporters: [reporter] of Crytic::Reporter::Reporter, - no_mutation_factory: fake_no_mutation_factory) - - runner.run( - subjects(["./fixtures/require_order/blog.cr", "./fixtures/require_order/pages/blog/archive.cr"]), - ["./fixtures/simple/bar_spec.cr"]).should eq false + it "returns the runs final result" do + run = FakeRun.new + run.final_result = true + + Sequential.new.run(run, side_effects).should eq true + + run.final_result = false + Sequential.new.run(run, side_effects).should eq false end - it "doesn't execute mutations if the initial suite run fails" do - reporter = FakeReporter.new - runner = Sequential.new( - threshold: 100.0, - generator: FakeGenerator.new, - reporters: [reporter] of Crytic::Reporter::Reporter, - no_mutation_factory: ->(specs : Array(String)) { - process_runner = Crytic::FakeProcessRunner.new - no_mutation = Crytic::Mutation::NoMutation.with(specs, process_runner) - process_runner.exit_code = [1, 0] - no_mutation - }) - - runner.run( - subjects(["./fixtures/require_order/blog.cr", "./fixtures/require_order/pages/blog/archive.cr"]), - ["./fixtures/simple/bar_spec.cr"]).should eq false + it "returns false if the original spec suite fails" do + run = FakeRun.new + run.original_exit_code = 1 + + Sequential.new.run(run, side_effects).should eq false end - it "reports events in order" do - reporter = FakeReporter.new - runner = Sequential.new( - threshold: 100.0, - generator: FakeGenerator.new([fake_mutation]), - reporters: [reporter] of Crytic::Reporter::Reporter, - no_mutation_factory: fake_no_mutation_factory) + it "reports neutral results before mutation results" do + run = FakeRun.new + run.mutations = [FakeMutation.new.as(Crytic::Mutation::Mutation)] - runner.run(subjects(["./fixtures/simple/bar.cr"]), ["./fixtures/simple/bar_spec.cr"]) + Sequential.new.run(run, side_effects) - reporter.events.should eq ["report_original_result", "report_mutations", "report_neutral_result", "report_result", "report_summary", "report_msi"] + run.events.should eq ["report_neutral_result", "report_result"] end it "skips the mutations if the neutral result errored" do - reporter = FakeReporter.new + run = FakeRun.new mutation = fake_mutation - runner = Sequential.new( - threshold: 100.0, - generator: FakeGenerator.new( - neutral: erroring_mutation, - mutations: [mutation]), - reporters: [reporter] of Crytic::Reporter::Reporter, - no_mutation_factory: fake_no_mutation_factory) + run.neutral = FakeMutation.new(Crytic::Mutation::Status::Errored) + run.mutations = [mutation] - runner.run(subjects(["./fixtures/simple/bar.cr"]), ["./fixtures/simple/bar_spec.cr"]) + Sequential.new.run(run, side_effects) - reporter.events.should_not contain("report_result") + run.events.should_not contain("report_result") mutation.as(FakeMutation).run_call_count.should eq 0 end end end end -private def runner - Crytic::Runner::Sequential.new( - threshold: 100.0, - reporters: [] of Crytic::Reporter::Reporter, - generator: FakeGenerator.new) +private class FakeRun + property mutations = [] of Crytic::Mutation::Mutation + property events = [] of String + property original_exit_code = 0 + property final_result = true + property neutral = FakeMutation.new.as(Crytic::Mutation::Mutation) + + def generate_mutations + [Crytic::Generator::MutationSet.new(neutral, mutations)] + end + + def report_neutral_result(result) + events << "report_neutral_result" + end + + def report_result(result) + events << "report_result" + end + + def report_final(results) + final_result + end + + def execute_original_test_suite(side_effects) + Crytic::Mutation::OriginalResult.new(exit_code: original_exit_code, output: "") + end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 41ce3214..879e0b7f 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -6,7 +6,6 @@ require "./fake_generator" require "./fake_http_client" require "./fake_mutation" require "./fake_process_runner" -require "./fake_reporter" require "compiler/crystal/syntax/*" require "spec" @@ -99,7 +98,7 @@ end def fake_no_mutation_factory ->(specs : Array(String)) { - Crytic::Mutation::NoMutation.with(specs, Crytic::FakeProcessRunner.new) + Crytic::Mutation::NoMutation.with(specs) } end diff --git a/src/crytic/command/test.cr b/src/crytic/command/test.cr index 818dab8e..97cd7754 100644 --- a/src/crytic/command/test.cr +++ b/src/crytic/command/test.cr @@ -12,10 +12,13 @@ class Crytic::Command::Test def execute(args) options = parse_options(args) generator = build_generator(options) + factory = ->(specs : Array(String)) { + Mutation::NoMutation.with(specs) + } Crytic::Runner::Sequential - .new(options.msi_threshold, options.reporters, generator) - .run(options.subject, options.spec_files) + .new + .run(Crytic::Runner::Run.from_options(options, generator, factory), @side_effects) end private def parse_options(args) diff --git a/src/crytic/mutation/no_mutation.cr b/src/crytic/mutation/no_mutation.cr index 4a3839d3..901de9b8 100644 --- a/src/crytic/mutation/no_mutation.cr +++ b/src/crytic/mutation/no_mutation.cr @@ -4,19 +4,18 @@ require "./original_result" module Crytic::Mutation class NoMutation - def run + def run(side_effects) io = IO::Memory.new args = ["spec"] + @specs_file_paths - exit_code = @process_runner.run("crystal", args, output: io, error: io) + exit_code = side_effects.execute("crystal", args, output: io, error: io) OriginalResult.new(exit_code: exit_code, output: io.to_s) end - def self.with(specs : Array(String), process_runner) - new(specs, process_runner) + def self.with(specs : Array(String)) + new(specs) end - private def initialize(@specs_file_paths : Array(String), - @process_runner : ProcessRunner) + private def initialize(@specs_file_paths : Array(String)) end end end diff --git a/src/crytic/reporter/reporter.cr b/src/crytic/reporter/reporter.cr index 07a4387d..3e239c82 100644 --- a/src/crytic/reporter/reporter.cr +++ b/src/crytic/reporter/reporter.cr @@ -10,4 +10,6 @@ module Crytic::Reporter abstract def report_summary(results : Mutation::ResultSet) abstract def report_msi(results : Mutation::ResultSet) end + + alias Reporters = Array(Reporter) end diff --git a/src/crytic/runner/run.cr b/src/crytic/runner/run.cr new file mode 100644 index 00000000..798c794d --- /dev/null +++ b/src/crytic/runner/run.cr @@ -0,0 +1,48 @@ +require "../mutation/no_mutation" +require "../reporter/reporter" +require "../subject" + +module Crytic::Runner + alias NoMutationFactory = (Array(String)) -> Mutation::NoMutation + + class Run + def initialize( + @msi_threshold : Float64, + @reporters : Crytic::Reporter::Reporters, + @spec_files : Array(String), + @subjects : Array(Subject), + @generator : Generator::Generator, + @no_mutation_factory : NoMutationFactory + ) + end + + def self.from_options(options, generator, no_mutation_factory) + new(options.msi_threshold, options.reporters, options.spec_files, options.subject, generator, no_mutation_factory) + end + + def execute_original_test_suite(side_effects) + original_result = @no_mutation_factory.call(@spec_files).run(side_effects) + report_original_result(original_result) + original_result + end + + def generate_mutations + mutations = @generator.mutations_for(@subjects, @spec_files) + report_mutations(mutations) + mutations + end + + {% for method in [:original_result, :mutations, :neutral_result, :result, :msi, :summary] %} + def report_{{ method.id }}(result) + @reporters.each(&.report_{{ method.id }}(result)) + end + {% end %} + + def report_final(results) + report_summary(results) + report_msi(results) + + !results.empty? && MsiCalculator.new(results).msi.passes?(@msi_threshold) + end + end +end diff --git a/src/crytic/runner/sequential.cr b/src/crytic/runner/sequential.cr index 1844d789..1939efef 100644 --- a/src/crytic/runner/sequential.cr +++ b/src/crytic/runner/sequential.cr @@ -1,55 +1,24 @@ -require "../generator/generator" -require "../msi_calculator" -require "../mutation/no_mutation" require "../mutation/result" require "../mutation/result_set" -require "../reporter/reporter" +require "./run" module Crytic::Runner class Sequential - alias Threshold = Float64 - alias NoMutationFactory = (Array(String)) -> Mutation::NoMutation - - def initialize( - @threshold : Threshold, - @reporters : Array(Reporter::Reporter), - @generator : Generator::Generator, - @no_mutation_factory : NoMutationFactory = ->(specs : Array(String)) { - Mutation::NoMutation.with(specs, ProcessProcessRunner.new) - } - ) - end - - def run(subjects : Array(Subject), specs : Array(String)) : Bool - original_result = run_original_test_suite(specs) + def run(run, side_effects) : Bool + original_result = run.execute_original_test_suite(side_effects) return false unless original_result.successful? - mutations = determine_possible_mutations(subjects, specs) - results = Mutation::ResultSet.new(run_all_mutations(mutations)) - - @reporters.each(&.report_summary(results)) - @reporters.each(&.report_msi(results)) - - !results.empty? && MsiCalculator.new(results).msi.passes?(@threshold) - end - - private def run_original_test_suite(specs) - original_result = @no_mutation_factory.call(specs).run - @reporters.each(&.report_original_result(original_result)) - original_result - end + mutations = run.generate_mutations + results = Mutation::ResultSet.new(run_all_mutations(mutations, run)) - private def determine_possible_mutations(subject, specs) - mutations = @generator.mutations_for(subject, specs) - @reporters.each(&.report_mutations(mutations)) - mutations + run.report_final(results) end - private def run_mutations_for_single_subject(mutation_set) + private def run_mutations_for_single_subject(mutation_set, run) mutation_set.mutated.map do |mutation| result = mutation.run - @reporters.each(&.report_result(result)) + run.report_result(result) result end end @@ -58,15 +27,15 @@ module Crytic::Runner [] of Mutation::Result end - private def run_all_mutations(mutations) + private def run_all_mutations(mutations, run) mutations.map do |mutation_set| neutral_result = mutation_set.neutral.run - @reporters.each(&.report_neutral_result(neutral_result)) + run.report_neutral_result(neutral_result) if neutral_result.errored? discard_further_mutations_for_single_subject else - run_mutations_for_single_subject(mutation_set) + run_mutations_for_single_subject(mutation_set, run) end end.flatten end