From 3eed97d6d5e529f110712adf171657aaa47fc33d Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Sun, 27 Oct 2019 12:13:31 +0100
Subject: [PATCH 1/9] Bump to crystal 0.31.1
---
.circleci/config.yml | 2 +-
shard.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6a13a208..3565b1eb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,7 +3,7 @@ version: 2
jobs:
test-crystal: &test-template
docker:
- - image: crystallang/crystal:0.29.0
+ - image: crystallang/crystal:0.31.1
steps:
- checkout
test:
diff --git a/shard.yml b/shard.yml
index 7c9c8c4a..f36b36b8 100644
--- a/shard.yml
+++ b/shard.yml
@@ -4,7 +4,7 @@ version: 6.0.0
authors:
- Hannes Käufler <hannes.kaeufler@gmail.com>
-crystal: 0.29.0
+crystal: 0.31.1
license: MIT
From 773f298cbbcb3767f079fc623a85b64fb2af3fd8 Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Sun, 27 Oct 2019 12:16:46 +0100
Subject: [PATCH 2/9] Run formatter
---
spec/integration_spec.cr | 146 +++++++++++++++++++--------------------
1 file changed, 73 insertions(+), 73 deletions(-)
diff --git a/spec/integration_spec.cr b/spec/integration_spec.cr
index 08a03e5e..301bcc05 100644
--- a/spec/integration_spec.cr
+++ b/spec/integration_spec.cr
@@ -1,96 +1,96 @@
require "./spec_helper"
{% unless flag?("skip-integration") %}
-describe Crytic do
- describe "--help/-h" do
- it "prints usage info" do
- result = run_crytic("--help")
- result.output.should contain("Usage: crytic [arguments]")
- result.exit_code.should eq 0
- result = run_crytic("-h")
- result.output.should contain("Usage: crytic [arguments]")
- result.exit_code.should eq 0
+ describe Crytic do
+ describe "--help/-h" do
+ it "prints usage info" do
+ result = run_crytic("--help")
+ result.output.should contain("Usage: crytic [arguments]")
+ result.exit_code.should eq 0
+ result = run_crytic("-h")
+ result.output.should contain("Usage: crytic [arguments]")
+ result.exit_code.should eq 0
+ end
end
- end
- describe "--preamble/-p" do
- it "injects the given custom preamble, failing the neutral mutant" do
- result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr -p 'exit 1'")
- result.output.should contain("unmodified subject")
- result.output.should_not contain("ConditionFlip")
- result.exit_code.should eq 1
+ describe "--preamble/-p" do
+ it "injects the given custom preamble, failing the neutral mutant" do
+ result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr -p 'exit 1'")
+ result.output.should contain("unmodified subject")
+ result.output.should_not contain("ConditionFlip")
+ result.exit_code.should eq 1
+ end
end
- end
- describe "with a fully covered subject" do
- it "passes the mutation specs" do
- result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/fully_covered_spec.cr")
- result.output.should contain("✅ ConditionFlip")
- result.output.should contain("✅ BoolLiteralFlip")
- result.output.should contain("3 covered")
- result.exit_code.should eq 0
+ describe "with a fully covered subject" do
+ it "passes the mutation specs" do
+ result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/fully_covered_spec.cr")
+ result.output.should contain("✅ ConditionFlip")
+ result.output.should contain("✅ BoolLiteralFlip")
+ result.output.should contain("3 covered")
+ result.exit_code.should eq 0
+ end
end
- end
- describe "with an insufficiently covered subject" do
- it "fails the mutation specs" do
- result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr")
- result.output.should contain("❌ ConditionFlip")
- result.output.should contain("❌ BoolLiteralFlip")
- result.output.should contain("3 uncovered")
- result.exit_code.should be > 0
- end
+ describe "with an insufficiently covered subject" do
+ it "fails the mutation specs" do
+ result = run_crytic("-s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr")
+ result.output.should contain("❌ ConditionFlip")
+ result.output.should contain("❌ BoolLiteralFlip")
+ result.output.should contain("3 uncovered")
+ result.exit_code.should be > 0
+ end
- it "exits successfully when the msi threshold is set sufficiently" do
- result = run_crytic("--min-msi=0.0 -s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr")
- result.exit_code.should eq 0
+ it "exits successfully when the msi threshold is set sufficiently" do
+ result = run_crytic("--min-msi=0.0 -s ./fixtures/conditionals/fully_covered.cr ./fixtures/conditionals/uncovered_spec.cr")
+ result.exit_code.should eq 0
+ end
end
- end
- describe "without passing a subject or tests" do
- it "mutates all sources and runs all tests" do
- result = run_crytic_in_dir("./fixtures/autofind")
- result.output.should contain("✅ ConditionFlip")
- result.output.should contain("✅ BoolLiteralFlip")
- result.output.should contain("✅ NumberLiteralSignFlip")
- result.output.should contain("✅ NumberLiteralChange")
- result.output.should contain("❌ NumberLiteralSignFlip")
- result.output.should contain("❌ NumberLiteralChange")
- result.output.should contain("2 uncovered")
- result.exit_code.should be > 0
+ describe "without passing a subject or tests" do
+ it "mutates all sources and runs all tests" do
+ result = run_crytic_in_dir("./fixtures/autofind")
+ result.output.should contain("✅ ConditionFlip")
+ result.output.should contain("✅ BoolLiteralFlip")
+ result.output.should contain("✅ NumberLiteralSignFlip")
+ result.output.should contain("✅ NumberLiteralChange")
+ result.output.should contain("❌ NumberLiteralSignFlip")
+ result.output.should contain("❌ NumberLiteralChange")
+ result.output.should contain("2 uncovered")
+ result.exit_code.should be > 0
+ end
end
- end
- describe "subject without any coverage" do
- it "fails all mutants" do
- result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/uncovered/without_spec.cr")
- result.output.should contain("❌ BoolLiteralFlip")
- result.output.should contain("❌ ConditionFlip")
- result.output.should contain("❌ NumberLiteralSignFlip")
- result.output.should contain("❌ NumberLiteralChange")
- result.output.should contain("9 uncovered")
- result.exit_code.should be > 0
+ describe "subject without any coverage" do
+ it "fails all mutants" do
+ result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/uncovered/without_spec.cr")
+ result.output.should contain("❌ BoolLiteralFlip")
+ result.output.should contain("❌ ConditionFlip")
+ result.output.should contain("❌ NumberLiteralSignFlip")
+ result.output.should contain("❌ NumberLiteralChange")
+ result.output.should contain("9 uncovered")
+ result.exit_code.should be > 0
+ end
end
- end
- describe "a failing initial test suite" do
- it "reports initial failure" do
- result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/failing/failing_spec.cr")
- result.output.should contain "❌ Original test suite failed.\n"
- result.output.should contain "no overload matches"
- result.exit_code.should be > 0
+ describe "a failing initial test suite" do
+ it "reports initial failure" do
+ result = run_crytic("-s ./fixtures/uncovered/without.cr ./fixtures/failing/failing_spec.cr")
+ result.output.should contain "❌ Original test suite failed.\n"
+ result.output.should contain "no overload matches"
+ result.exit_code.should be > 0
+ end
end
- end
- describe "a subject that is mutated into an endless loop" do
- it "finishes and reports a timed out spec" do
- result = run_crytic("-s ./fixtures/timeout/timeout.cr ./fixtures/timeout/timeout_spec.cr")
- result.output.should contain "✅ Original test suite passed.\n"
- result.output.should contain "1 timeout"
- result.exit_code.should be > 0
+ describe "a subject that is mutated into an endless loop" do
+ it "finishes and reports a timed out spec" do
+ result = run_crytic("-s ./fixtures/timeout/timeout.cr ./fixtures/timeout/timeout_spec.cr")
+ result.output.should contain "✅ Original test suite passed.\n"
+ result.output.should contain "1 timeout"
+ result.exit_code.should be > 0
+ end
end
end
-end
{% end %}
def run_crytic_in_dir(dir : String)
From 134769b0fcf409376c5162654209999882e65180 Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Sun, 27 Oct 2019 12:25:35 +0100
Subject: [PATCH 3/9] Remove deprecation warnings by explicitly stating
overloaded methods return types
---
spec/fake_generator.cr | 2 +-
spec/fake_mutation.cr | 2 +-
src/crytic/generator/in_memory_generator.cr | 2 +-
src/crytic/mutation/isolated_mutation.cr | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/spec/fake_generator.cr b/spec/fake_generator.cr
index 89e842ef..696fd6f3 100644
--- a/spec/fake_generator.cr
+++ b/spec/fake_generator.cr
@@ -6,7 +6,7 @@ class FakeGenerator < Crytic::Generator::Generator
@neutral = FakeMutation.new.as(Crytic::Mutation::Mutation))
end
- def mutations_for(source : Array(Crytic::Subject), specs : Array(String))
+ def mutations_for(source : Array(Crytic::Subject), specs : Array(String)) : Array(Crytic::Generator::MutationSet)
[Crytic::Generator::MutationSet.new(
neutral: @neutral,
mutated: @mutations)]
diff --git a/spec/fake_mutation.cr b/spec/fake_mutation.cr
index c9f4c763..8a986eb6 100644
--- a/spec/fake_mutation.cr
+++ b/spec/fake_mutation.cr
@@ -16,7 +16,7 @@ class FakeMutation < Crytic::Mutation::Mutation
def initialize(@reported_status = Crytic::Mutation::Status::Uncovered)
end
- def run
+ def run : Crytic::Mutation::Result
@run_call_count += 1
Crytic::Mutation::Result.new(@reported_status, irrelevant_mutant, "", "")
end
diff --git a/src/crytic/generator/in_memory_generator.cr b/src/crytic/generator/in_memory_generator.cr
index 768e468b..18539509 100644
--- a/src/crytic/generator/in_memory_generator.cr
+++ b/src/crytic/generator/in_memory_generator.cr
@@ -15,7 +15,7 @@ module Crytic::Generator
)
end
- def mutations_for(sources : Array(Subject), specs : Array(String))
+ def mutations_for(sources : Array(Subject), specs : Array(String)) : Array(MutationSet)
sources
.map do |src|
MutationSet.new(
diff --git a/src/crytic/mutation/isolated_mutation.cr b/src/crytic/mutation/isolated_mutation.cr
index 2cfb6bb7..004844b1 100644
--- a/src/crytic/mutation/isolated_mutation.cr
+++ b/src/crytic/mutation/isolated_mutation.cr
@@ -14,7 +14,7 @@ module Crytic::Mutation
# Compiles the mutated source code into a binary and runs this binary,
# recording exit code, stderr and stdout output.
- def run
+ def run : Result
mutated = @environment.perform_mutation
process_result = run(mutated)
success_messages_in_output = /Finished/ =~ process_result[:output]
From c4bcd6f0fb81838009f10a665094c01eeafafefb Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Sun, 27 Oct 2019 12:25:46 +0100
Subject: [PATCH 4/9] Avoid deprecated Time.now
---
src/crytic/reporter/io_reporter.cr | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/crytic/reporter/io_reporter.cr b/src/crytic/reporter/io_reporter.cr
index dfcb9770..182c07ed 100644
--- a/src/crytic/reporter/io_reporter.cr
+++ b/src/crytic/reporter/io_reporter.cr
@@ -8,7 +8,7 @@ module Crytic::Reporter
class IoReporter < Reporter
INDENT = " "
- def initialize(@io : IO, @start_time = Time.now)
+ def initialize(@io : IO, @start_time = Time.utc)
end
def report_original_result(original_result)
@@ -77,7 +77,7 @@ module Crytic::Reporter
end
private def elapsed_time
- Time.now - @start_time
+ Time.utc - @start_time
end
end
end
From 968326578d0500419b5f237c2f9c1941dcd6a84b Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Fri, 1 Nov 2019 20:48:06 +0100
Subject: [PATCH 5/9] Ooof. Be more lax about the expected output.
We have a default preamble which enables
fail fast mode in specs. Between crystal 0.30.1
and 0.31.0 this changed the output. The new crystal
doesnt print a summary anymore, which means we will
not find "Finished" in it. Instead only one "F" is printed and
that's it (if there is exactly one test which fails).
See also https://github.com/crystal-lang/crystal/issues/8359
---
src/crytic/mutation/isolated_mutation.cr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/crytic/mutation/isolated_mutation.cr b/src/crytic/mutation/isolated_mutation.cr
index 004844b1..78f3f1d4 100644
--- a/src/crytic/mutation/isolated_mutation.cr
+++ b/src/crytic/mutation/isolated_mutation.cr
@@ -17,7 +17,7 @@ module Crytic::Mutation
def run : Result
mutated = @environment.perform_mutation
process_result = run(mutated)
- success_messages_in_output = /Finished/ =~ process_result[:output]
+ success_messages_in_output = /F/ =~ process_result[:output]
status = if process_result[:exit_code] == ProcessRunner::SUCCESS
Status::Uncovered
elsif process_result[:exit_code] == ProcessRunner::TIMEOUT
From bc4720ebac4c4d1715632584dbdc26bccbbe9291 Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Fri, 1 Nov 2019 22:32:41 +0100
Subject: [PATCH 6/9] Hah, even the exit code is wrong. Disable fail fast mode
alltogether. There is another bug in crystal which makes the fail fast mode
entirely unusable for crystal.
---
spec/cli_options_spec.cr | 8 ++++++++
src/crytic/cli_options.cr | 21 ++++++++++++++++++++-
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/spec/cli_options_spec.cr b/spec/cli_options_spec.cr
index 863d57ff..d07b6ba1 100644
--- a/spec/cli_options_spec.cr
+++ b/spec/cli_options_spec.cr
@@ -98,11 +98,19 @@ module Crytic
end
{% end %}
+ {% if Crystal::VERSION == "0.31.0" || Crystal::VERSION == "0.31.1" %}
+ it "defaults to a simple spec preamble" do
+ opts = cli_options_parser.parse([] of String)
+
+ opts.preamble.should eq "require \"spec\"\n\n"
+ end
+ {% else %}
it "defaults to a fail fast preamble" do
opts = cli_options_parser.parse([] of String)
opts.preamble.should eq Generator::Generator::DEFAULT_PREAMBLE
end
+ {% end %}
{% for flag in ["-m", "--min-msi"] %}
it "accepts a msi threshold with {{ flag.id }}" do
diff --git a/src/crytic/cli_options.cr b/src/crytic/cli_options.cr
index d07fe82d..88370df7 100644
--- a/src/crytic/cli_options.cr
+++ b/src/crytic/cli_options.cr
@@ -2,6 +2,7 @@ require "./generator/generator"
require "./mutant/possibilities"
require "./reporter/*"
require "./side_effects"
+require "colorize"
require "option_parser"
module Crytic
@@ -9,8 +10,8 @@ module Crytic
DEFAULT_SPEC_FILES_GLOB = "./spec/**/*_spec.cr"
getter msi_threshold = 100.0
getter mutants : Array(Mutant::Possibilities) = Generator::Generator::ALL_MUTANTS
- getter preamble = Generator::Generator::DEFAULT_PREAMBLE
getter reporters = [] of Reporter::Reporter
+ @preamble = Generator::Generator::DEFAULT_PREAMBLE
@spec_files = [] of String
@subject = [] of String
@@ -87,6 +88,24 @@ module Crytic
end.map { |path| Subject.from_filepath(path) }
end
+ def preamble
+ {% if Crystal::VERSION == "0.31.1" || Crystal::VERSION == "0.31.0" %}
+ if @preamble =~ /fail_fast/
+ puts(<<-MSG.colorize(:yellow))
+
+ ⚠️ You specified a preamble that uses Spec#fail_fast. Due to a bug
+ in crystal 0.31.0 and 0.31.1, the Spec#fail_fast mode cannot be
+ used in crytic. We disabled it for you. See
+ https://github.com/crystal-lang/crystal/issues/8420 for details.
+
+ MSG
+ return @preamble.gsub(/Spec\.fail_fast\s*=\s*true/, "")
+ end
+ {% end %}
+
+ @preamble
+ end
+
private def console_reporter
Reporter::IoReporter.new(@side_effects.std_out)
end
From 10073428493d07b17cfafae6ad6563b2c1d74569 Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Fri, 1 Nov 2019 22:36:37 +0100
Subject: [PATCH 7/9] Run tests on both crystal 0.30.1 and 0.31.1
---
.circleci/config.yml | 16 +++++++++++++---
spec/cli_options_spec.cr | 16 ++++++++--------
2 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3565b1eb..f5d4b34b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,15 @@ jobs:
- image: crystallang/crystal:0.31.1
steps:
- checkout
- test:
+ test-crystal-0.30.1:
+ <<: *test-template
+ docker:
+ - image: crystallang/crystal:0.30.1
+ steps:
+ - checkout
+ - run: shards
+ - run: SPEC_VERBOSE=1 ./bin/test
+ test-crystal-0.31.1:
<<: *test-template
steps:
- checkout
@@ -29,7 +37,8 @@ workflows:
version: 2
ci:
jobs:
- - test
+ - test-crystal-0.31.1
+ - test-crystal-0.30.1
nightly:
triggers:
- schedule:
@@ -38,5 +47,6 @@ workflows:
branches:
only: master
jobs:
- - test
+ - test-crystal-0.31.1
+ - test-crystal-0.30.1
- test-mutations
diff --git a/spec/cli_options_spec.cr b/spec/cli_options_spec.cr
index d07b6ba1..f1b46723 100644
--- a/spec/cli_options_spec.cr
+++ b/spec/cli_options_spec.cr
@@ -99,17 +99,17 @@ module Crytic
{% end %}
{% if Crystal::VERSION == "0.31.0" || Crystal::VERSION == "0.31.1" %}
- it "defaults to a simple spec preamble" do
- opts = cli_options_parser.parse([] of String)
+ it "defaults to a simple spec preamble" do
+ opts = cli_options_parser.parse([] of String)
- opts.preamble.should eq "require \"spec\"\n\n"
- end
+ opts.preamble.should eq "require \"spec\"\n\n"
+ end
{% else %}
- it "defaults to a fail fast preamble" do
- opts = cli_options_parser.parse([] of String)
+ it "defaults to a fail fast preamble" do
+ opts = cli_options_parser.parse([] of String)
- opts.preamble.should eq Generator::Generator::DEFAULT_PREAMBLE
- end
+ opts.preamble.should eq Generator::Generator::DEFAULT_PREAMBLE
+ end
{% end %}
{% for flag in ["-m", "--min-msi"] %}
From 9f1d89e161845d43d313a2ea421f6c146f83cd0a Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Fri, 1 Nov 2019 22:42:20 +0100
Subject: [PATCH 8/9] Add changelog for 0.31.x compat
---
CHANGELOG.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bab29ce7..d915c2c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] - Unknown
+- Crystal 0.31.x compatibility (no changes were needed for crystal 0.30.x). Be careful
+ with crystal 0.31.0 and 0.31.1 because there is a bug that might cause your CI job
+ to pass even with failing tests. See [#8420](https://github.com/crystal-lang/crystal/issues/8420).
+
## [6.0.0] - 2019-06-30
### Changed
From 6622ce7309889d4a8693a805ab0e5fd98b8851db Mon Sep 17 00:00:00 2001
From: Hannes Kaeufler <hannes.kaeufler@gmail.com>
Date: Fri, 1 Nov 2019 23:22:18 +0100
Subject: [PATCH 9/9] Simplify conditional spec
---
spec/cli_options_spec.cr | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/spec/cli_options_spec.cr b/spec/cli_options_spec.cr
index f1b46723..cdd28153 100644
--- a/spec/cli_options_spec.cr
+++ b/spec/cli_options_spec.cr
@@ -98,19 +98,15 @@ module Crytic
end
{% end %}
- {% if Crystal::VERSION == "0.31.0" || Crystal::VERSION == "0.31.1" %}
- it "defaults to a simple spec preamble" do
- opts = cli_options_parser.parse([] of String)
+ it "defaults to a fail fast preamble" do
+ opts = cli_options_parser.parse([] of String)
+ {% if Crystal::VERSION == "0.31.0" || Crystal::VERSION == "0.31.1" %}
opts.preamble.should eq "require \"spec\"\n\n"
- end
- {% else %}
- it "defaults to a fail fast preamble" do
- opts = cli_options_parser.parse([] of String)
-
+ {% else %}
opts.preamble.should eq Generator::Generator::DEFAULT_PREAMBLE
- end
- {% end %}
+ {% end %}
+ end
{% for flag in ["-m", "--min-msi"] %}
it "accepts a msi threshold with {{ flag.id }}" do