diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a13a208..f5d4b34b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,10 +3,18 @@ version: 2 jobs: test-crystal: &test-template docker: - - image: crystallang/crystal:0.29.0 + - image: crystallang/crystal:0.31.1 steps: - checkout - test: + test-crystal-0.30.1: + <<: *test-template + docker: + - image: crystallang/crystal:0.30.1 + steps: + - checkout + - run: shards + - run: SPEC_VERBOSE=1 ./bin/test + test-crystal-0.31.1: <<: *test-template steps: - checkout @@ -29,7 +37,8 @@ workflows: version: 2 ci: jobs: - - test + - test-crystal-0.31.1 + - test-crystal-0.30.1 nightly: triggers: - schedule: @@ -38,5 +47,6 @@ workflows: branches: only: master jobs: - - test + - test-crystal-0.31.1 + - test-crystal-0.30.1 - test-mutations diff --git a/CHANGELOG.md b/CHANGELOG.md index bab29ce7..d915c2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Unknown +- Crystal 0.31.x compatibility (no changes were needed for crystal 0.30.x). Be careful + with crystal 0.31.0 and 0.31.1 because there is a bug that might cause your CI job + to pass even with failing tests. See [#8420](https://github.com/crystal-lang/crystal/issues/8420). + ## [6.0.0] - 2019-06-30 ### Changed diff --git a/shard.yml b/shard.yml index 7c9c8c4a..f36b36b8 100644 --- a/shard.yml +++ b/shard.yml @@ -4,7 +4,7 @@ version: 6.0.0 authors: - Hannes Käufler -crystal: 0.29.0 +crystal: 0.31.1 license: MIT diff --git a/spec/cli_options_spec.cr b/spec/cli_options_spec.cr index 863d57ff..cdd28153 100644 --- a/spec/cli_options_spec.cr +++ b/spec/cli_options_spec.cr @@ -101,7 +101,11 @@ module Crytic it "defaults to a fail fast preamble" do opts = cli_options_parser.parse([] of String) - opts.preamble.should eq Generator::Generator::DEFAULT_PREAMBLE + {% if Crystal::VERSION == "0.31.0" || Crystal::VERSION == "0.31.1" %} + opts.preamble.should eq "require \"spec\"\n\n" + {% else %} + opts.preamble.should eq Generator::Generator::DEFAULT_PREAMBLE + {% end %} end {% for flag in ["-m", "--min-msi"] %} diff --git a/spec/fake_generator.cr b/spec/fake_generator.cr index 89e842ef..696fd6f3 100644 --- a/spec/fake_generator.cr +++ b/spec/fake_generator.cr @@ -6,7 +6,7 @@ class FakeGenerator < Crytic::Generator::Generator @neutral = FakeMutation.new.as(Crytic::Mutation::Mutation)) end - def mutations_for(source : Array(Crytic::Subject), specs : Array(String)) + def mutations_for(source : Array(Crytic::Subject), specs : Array(String)) : Array(Crytic::Generator::MutationSet) [Crytic::Generator::MutationSet.new( neutral: @neutral, mutated: @mutations)] diff --git a/spec/fake_mutation.cr b/spec/fake_mutation.cr index c9f4c763..8a986eb6 100644 --- a/spec/fake_mutation.cr +++ b/spec/fake_mutation.cr @@ -16,7 +16,7 @@ class FakeMutation < Crytic::Mutation::Mutation def initialize(@reported_status = Crytic::Mutation::Status::Uncovered) end - def run + def run : Crytic::Mutation::Result @run_call_count += 1 Crytic::Mutation::Result.new(@reported_status, irrelevant_mutant, "", "") end diff --git a/spec/integration_spec.cr b/spec/integration_spec.cr index 08a03e5e..301bcc05 100644 --- a/spec/integration_spec.cr +++ b/spec/integration_spec.cr @@ -1,96 +1,96 @@ require "./spec_helper" {% unless flag?("skip-integration") %} -describe Crytic do - describe "--help/-h" do - it "prints usage info" do - result = run_crytic("--help") - result.output.should contain("Usage: crytic [arguments]") - result.exit_code.should eq 0 - result = run_crytic("-h") - result.output.should contain("Usage: crytic [arguments]") - result.exit_code.should eq 0 + describe Crytic do + describe "--help/-h" do + it "prints usage info" do + result = run_crytic("--help") + result.output.should contain("Usage: crytic [arguments]") + result.exit_code.should eq 0 + result = run_crytic("-h") + result.output.should contain("Usage: crytic [arguments]") + result.exit_code.should eq 0 + end end - end - describe "--preamble/-p" do - it "injects the given custom preamble, failing the neutral mutant" do - result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr -p 'exit 1'") - result.output.should contain("unmodified subject") - result.output.should_not contain("ConditionFlip") - result.exit_code.should eq 1 + describe "--preamble/-p" do + it "injects the given custom preamble, failing the neutral mutant" do + result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr -p 'exit 1'") + result.output.should contain("unmodified subject") + result.output.should_not contain("ConditionFlip") + result.exit_code.should eq 1 + end end - end - describe "with a fully covered subject" do - it "passes the mutation specs" do - result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/fully_covered_spec.cr") - result.output.should contain("✅ ConditionFlip") - result.output.should contain("✅ BoolLiteralFlip") - result.output.should contain("3 covered") - result.exit_code.should eq 0 + describe "with a fully covered subject" do + it "passes the mutation specs" do + result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/fully_covered_spec.cr") + result.output.should contain("✅ ConditionFlip") + result.output.should contain("✅ BoolLiteralFlip") + result.output.should contain("3 covered") + result.exit_code.should eq 0 + end end - end - describe "with an insufficiently covered subject" do - it "fails the mutation specs" do - result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr") - result.output.should contain("❌ ConditionFlip") - result.output.should contain("❌ BoolLiteralFlip") - result.output.should contain("3 uncovered") - result.exit_code.should be > 0 - end + describe "with an insufficiently covered subject" do + it "fails the mutation specs" do + result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr") + result.output.should contain("❌ ConditionFlip") + result.output.should contain("❌ BoolLiteralFlip") + result.output.should contain("3 uncovered") + result.exit_code.should be > 0 + end - it "exits successfully when the msi threshold is set sufficiently" do - result = run_crytic("--min-msi=0.0 -s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr") - result.exit_code.should eq 0 + it "exits successfully when the msi threshold is set sufficiently" do + result = run_crytic("--min-msi=0.0 -s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr") + result.exit_code.should eq 0 + end end - end - describe "without passing a subject or tests" do - it "mutates all sources and runs all tests" do - result = run_crytic_in_dir("./fixtures/autofind") - result.output.should contain("✅ ConditionFlip") - result.output.should contain("✅ BoolLiteralFlip") - result.output.should contain("✅ NumberLiteralSignFlip") - result.output.should contain("✅ NumberLiteralChange") - result.output.should contain("❌ NumberLiteralSignFlip") - result.output.should contain("❌ NumberLiteralChange") - result.output.should contain("2 uncovered") - result.exit_code.should be > 0 + describe "without passing a subject or tests" do + it "mutates all sources and runs all tests" do + result = run_crytic_in_dir("./fixtures/autofind") + result.output.should contain("✅ ConditionFlip") + result.output.should contain("✅ BoolLiteralFlip") + result.output.should contain("✅ NumberLiteralSignFlip") + result.output.should contain("✅ NumberLiteralChange") + result.output.should contain("❌ NumberLiteralSignFlip") + result.output.should contain("❌ NumberLiteralChange") + result.output.should contain("2 uncovered") + result.exit_code.should be > 0 + end end - end - describe "subject without any coverage" do - it "fails all mutants" do - result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/uncovered/without_spec.cr") - result.output.should contain("❌ BoolLiteralFlip") - result.output.should contain("❌ ConditionFlip") - result.output.should contain("❌ NumberLiteralSignFlip") - result.output.should contain("❌ NumberLiteralChange") - result.output.should contain("9 uncovered") - result.exit_code.should be > 0 + describe "subject without any coverage" do + it "fails all mutants" do + result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/uncovered/without_spec.cr") + result.output.should contain("❌ BoolLiteralFlip") + result.output.should contain("❌ ConditionFlip") + result.output.should contain("❌ NumberLiteralSignFlip") + result.output.should contain("❌ NumberLiteralChange") + result.output.should contain("9 uncovered") + result.exit_code.should be > 0 + end end - end - describe "a failing initial test suite" do - it "reports initial failure" do - result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/failing/failing_spec.cr") - result.output.should contain "❌ Original test suite failed.\n" - result.output.should contain "no overload matches" - result.exit_code.should be > 0 + describe "a failing initial test suite" do + it "reports initial failure" do + result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/failing/failing_spec.cr") + result.output.should contain "❌ Original test suite failed.\n" + result.output.should contain "no overload matches" + result.exit_code.should be > 0 + end end - end - describe "a subject that is mutated into an endless loop" do - it "finishes and reports a timed out spec" do - result = run_crytic("-s ./fixtures/timeout/timeout.cr ./fixtures/timeout/timeout_spec.cr") - result.output.should contain "✅ Original test suite passed.\n" - result.output.should contain "1 timeout" - result.exit_code.should be > 0 + describe "a subject that is mutated into an endless loop" do + it "finishes and reports a timed out spec" do + result = run_crytic("-s ./fixtures/timeout/timeout.cr ./fixtures/timeout/timeout_spec.cr") + result.output.should contain "✅ Original test suite passed.\n" + result.output.should contain "1 timeout" + result.exit_code.should be > 0 + end end end -end {% end %} def run_crytic_in_dir(dir : String) diff --git a/src/crytic/cli_options.cr b/src/crytic/cli_options.cr index d07fe82d..88370df7 100644 --- a/src/crytic/cli_options.cr +++ b/src/crytic/cli_options.cr @@ -2,6 +2,7 @@ require "./generator/generator" require "./mutant/possibilities" require "./reporter/*" require "./side_effects" +require "colorize" require "option_parser" module Crytic @@ -9,8 +10,8 @@ module Crytic DEFAULT_SPEC_FILES_GLOB = "./spec/**/*_spec.cr" getter msi_threshold = 100.0 getter mutants : Array(Mutant::Possibilities) = Generator::Generator::ALL_MUTANTS - getter preamble = Generator::Generator::DEFAULT_PREAMBLE getter reporters = [] of Reporter::Reporter + @preamble = Generator::Generator::DEFAULT_PREAMBLE @spec_files = [] of String @subject = [] of String @@ -87,6 +88,24 @@ module Crytic end.map { |path| Subject.from_filepath(path) } end + def preamble + {% if Crystal::VERSION == "0.31.1" || Crystal::VERSION == "0.31.0" %} + if @preamble =~ /fail_fast/ + puts(<<-MSG.colorize(:yellow)) + + ⚠️ You specified a preamble that uses Spec#fail_fast. Due to a bug + in crystal 0.31.0 and 0.31.1, the Spec#fail_fast mode cannot be + used in crytic. We disabled it for you. See + https://github.com/crystal-lang/crystal/issues/8420 for details. + + MSG + return @preamble.gsub(/Spec\.fail_fast\s*=\s*true/, "") + end + {% end %} + + @preamble + end + private def console_reporter Reporter::IoReporter.new(@side_effects.std_out) end diff --git a/src/crytic/generator/in_memory_generator.cr b/src/crytic/generator/in_memory_generator.cr index 768e468b..18539509 100644 --- a/src/crytic/generator/in_memory_generator.cr +++ b/src/crytic/generator/in_memory_generator.cr @@ -15,7 +15,7 @@ module Crytic::Generator ) end - def mutations_for(sources : Array(Subject), specs : Array(String)) + def mutations_for(sources : Array(Subject), specs : Array(String)) : Array(MutationSet) sources .map do |src| MutationSet.new( diff --git a/src/crytic/mutation/isolated_mutation.cr b/src/crytic/mutation/isolated_mutation.cr index 2cfb6bb7..78f3f1d4 100644 --- a/src/crytic/mutation/isolated_mutation.cr +++ b/src/crytic/mutation/isolated_mutation.cr @@ -14,10 +14,10 @@ module Crytic::Mutation # Compiles the mutated source code into a binary and runs this binary, # recording exit code, stderr and stdout output. - def run + def run : Result mutated = @environment.perform_mutation process_result = run(mutated) - success_messages_in_output = /Finished/ =~ process_result[:output] + success_messages_in_output = /F/ =~ process_result[:output] status = if process_result[:exit_code] == ProcessRunner::SUCCESS Status::Uncovered elsif process_result[:exit_code] == ProcessRunner::TIMEOUT diff --git a/src/crytic/reporter/io_reporter.cr b/src/crytic/reporter/io_reporter.cr index dfcb9770..182c07ed 100644 --- a/src/crytic/reporter/io_reporter.cr +++ b/src/crytic/reporter/io_reporter.cr @@ -8,7 +8,7 @@ module Crytic::Reporter class IoReporter < Reporter INDENT = " " - def initialize(@io : IO, @start_time = Time.now) + def initialize(@io : IO, @start_time = Time.utc) end def report_original_result(original_result) @@ -77,7 +77,7 @@ module Crytic::Reporter end private def elapsed_time - Time.now - @start_time + Time.utc - @start_time end end end