diff --git a/cmd/nerdctl/volume/volume_inspect_test.go b/cmd/nerdctl/volume/volume_inspect_test.go index 7e627a119b5..86d070065e4 100644 --- a/cmd/nerdctl/volume/volume_inspect_test.go +++ b/cmd/nerdctl/volume/volume_inspect_test.go @@ -18,7 +18,6 @@ package volume import ( "crypto/rand" - "encoding/json" "errors" "fmt" "os" @@ -31,6 +30,7 @@ import ( "github.com/containerd/nerdctl/mod/tigron/expect" "github.com/containerd/nerdctl/mod/tigron/require" "github.com/containerd/nerdctl/mod/tigron/test" + "github.com/containerd/nerdctl/mod/tigron/tig" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/native" "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" @@ -99,15 +99,11 @@ func TestVolumeInspect(t *testing.T) { return &test.Expected{ Output: expect.All( expect.Contains(data.Get("vol1")), - func(stdout string, info string, t *testing.T) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } + expect.JSON([]native.Volume{}, func(dc []native.Volume, info string, t tig.T) { assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))+info) assert.Assert(t, dc[0].Name == data.Get("vol1"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("vol1"), dc[0].Name)+info) assert.Assert(t, dc[0].Labels == nil, fmt.Sprintf("expected labels to be nil and were %v", dc[0].Labels)+info) - }, + }), ), } }, @@ -121,16 +117,12 @@ func TestVolumeInspect(t *testing.T) { return &test.Expected{ Output: expect.All( expect.Contains(data.Get("vol2")), - func(stdout string, info string, t *testing.T) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } + expect.JSON([]native.Volume{}, func(dc []native.Volume, info string, t tig.T) { labels := *dc[0].Labels assert.Assert(t, len(labels) == 2, fmt.Sprintf("two results, not %d", len(labels))) assert.Assert(t, labels["foo"] == "fooval", fmt.Sprintf("label foo should be fooval, not %s", labels["foo"])) assert.Assert(t, labels["bar"] == "barval", fmt.Sprintf("label bar should be barval, not %s", labels["bar"])) - }, + }), ), } }, @@ -145,13 +137,9 @@ func TestVolumeInspect(t *testing.T) { return &test.Expected{ Output: expect.All( expect.Contains(data.Get("vol1")), - func(stdout string, info string, t *testing.T) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } + expect.JSON([]native.Volume{}, func(dc []native.Volume, info string, t tig.T) { assert.Assert(t, dc[0].Size == size, fmt.Sprintf("expected size to be %d (was %d)", size, dc[0].Size)) - }, + }), ), } }, @@ -166,15 +154,11 @@ func TestVolumeInspect(t *testing.T) { Output: expect.All( expect.Contains(data.Get("vol1")), expect.Contains(data.Get("vol2")), - func(stdout string, info string, t *testing.T) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } + expect.JSON([]native.Volume{}, func(dc []native.Volume, info string, t tig.T) { assert.Assert(t, len(dc) == 2, fmt.Sprintf("two results, not %d", len(dc))) assert.Assert(t, dc[0].Name == data.Get("vol1"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("vol1"), dc[0].Name)) assert.Assert(t, dc[1].Name == data.Get("vol2"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("vol2"), dc[1].Name)) - }, + }), ), } }, @@ -190,14 +174,10 @@ func TestVolumeInspect(t *testing.T) { Errors: []error{errdefs.ErrNotFound, errdefs.ErrInvalidArgument}, Output: expect.All( expect.Contains(data.Get("vol1")), - func(stdout string, info string, t *testing.T) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } + expect.JSON([]native.Volume{}, func(dc []native.Volume, info string, t tig.T) { assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) assert.Assert(t, dc[0].Name == data.Get("vol1"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("vol1"), dc[0].Name)) - }, + }), ), } }, diff --git a/mod/tigron/expect/comparators.go b/mod/tigron/expect/comparators.go index 36b09de5e45..d7f944f32a8 100644 --- a/mod/tigron/expect/comparators.go +++ b/mod/tigron/expect/comparators.go @@ -19,11 +19,13 @@ package expect import ( + "encoding/json" "regexp" "testing" "github.com/containerd/nerdctl/mod/tigron/internal/assertive" "github.com/containerd/nerdctl/mod/tigron/test" + "github.com/containerd/nerdctl/mod/tigron/tig" ) // All can be used as a parameter for expected.Output to group a set of comparators. @@ -69,3 +71,18 @@ func Match(reg *regexp.Regexp) test.Comparator { assertive.Match(assertive.WithFailLater(t), stdout, reg, info) } } + +// JSON allows to verify that the output can be marshalled into T, and optionally can be further verified by a provided +// method. +func JSON[T any](obj T, verifier func(T, string, tig.T)) test.Comparator { + return func(stdout, info string, t *testing.T) { + t.Helper() + + err := json.Unmarshal([]byte(stdout), &obj) + assertive.ErrorIsNil(assertive.WithFailLater(t), err, "failed to unmarshal JSON from stdout") + + if verifier != nil && err == nil { + verifier(obj, info, t) + } + } +} diff --git a/mod/tigron/expect/comparators_test.go b/mod/tigron/expect/comparators_test.go index c6710f6073b..f216d3ecb51 100644 --- a/mod/tigron/expect/comparators_test.go +++ b/mod/tigron/expect/comparators_test.go @@ -20,24 +20,45 @@ package expect_test // TODO: add a lot more tests including failure conditions with mimicry import ( + "encoding/json" "regexp" "testing" "github.com/containerd/nerdctl/mod/tigron/expect" + "github.com/containerd/nerdctl/mod/tigron/internal/assertive" + "github.com/containerd/nerdctl/mod/tigron/tig" ) func TestExpect(t *testing.T) { t.Parallel() - expect.Contains("b")("a b c", "info", t) - expect.DoesNotContain("d")("a b c", "info", t) - expect.Equals("a b c")("a b c", "info", t) - expect.Match(regexp.MustCompile("[a-z ]+"))("a b c", "info", t) + expect.Contains("b")("a b c", "contains works", t) + expect.DoesNotContain("d")("a b c", "does not contain works", t) + expect.Equals("a b c")("a b c", "equals work", t) + expect.Match(regexp.MustCompile("[a-z ]+"))("a b c", "match works", t) expect.All( expect.Contains("b"), expect.DoesNotContain("d"), expect.Equals("a b c"), expect.Match(regexp.MustCompile("[a-z ]+")), - )("a b c", "info", t) + )("a b c", "all", t) + + type foo struct { + Foo map[string]string `json:"foo"` + } + + data, err := json.Marshal(&foo{ + Foo: map[string]string{ + "foo": "bar", + }, + }) + + assertive.ErrorIsNil(t, err) + + expect.JSON(&foo{}, nil)(string(data), "json, no verifier", t) + + expect.JSON(&foo{}, func(obj *foo, info string, t tig.T) { + assertive.IsEqual(t, obj.Foo["foo"], "bar", info) + })(string(data), "json, with verifier", t) } diff --git a/mod/tigron/expect/doc.md b/mod/tigron/expect/doc.md new file mode 100644 index 00000000000..48b8d4be6e9 --- /dev/null +++ b/mod/tigron/expect/doc.md @@ -0,0 +1,221 @@ +# Expectations + +Attaching expectations to a test case is how the developer can express conditions on exit code, stdout, or stderr, +to be verified for the test to pass. + +The simplest way to do that is to use the helper `test.Expects(exitCode int, errors []error, outputCompare test.Comparator)`. + +```go +package main + +import ( + "testing" + + "github.com/containerd/nerdctl/mod/tigron/test" +) + +func TestMyThing(t *testing.T) { + // Declare your test + myTest := &test.Case{} + + // Attach a command to run + myTest.Command = test.Custom("ls") + + // Set your expectations + myTest.Expected = test.Expects(expect.ExitCodeSuccess, nil, nil) + + // Run it + myTest.Run(t) +} +``` + +### Exit status expectations + +The first parameter, `exitCode` should be set to one of the provided `expect.ExitCodeXXX` constants: +- `expect.ExitCodeSuccess`: validates that the command ran and exited successfully +- `expect.ExitCodeTimeout`: validates that the command did time out +- `expect.ExitCodeSignaled`: validates that the command received a signal +- `expect.ExitCodeGenericFail`: validates that the command failed (failed to start, or returned a non-zero exit code) +- `expect.ExitCodeNoCheck`: does not enforce any verification at all on the command + +... you may also pass explicit exit codes directly (> 0) if you want to precisely match them. + +### Stderr expectations with []error + +To validate that stderr contain specific information, you can pass a slice of `error` as `test.Expects` +second parameter. + +The command output on stderr is then verified to contain all stringified errors. + +### Stdout expectations with Comparators + +The last parameter of `test.Expects` accepts a `test.Comparator`, which allows testing the content of the command +output on `stdout`. + +The following ready-made `test.Comparator` generators are provided: +- `expect.Contains(string)`: verifies that stdout contains the string parameter +- `expect.DoesNotContain(string)`: negation of above +- `expect.Equals(string)`: strict equality +- `expect.Match(*regexp.Regexp)`: regexp matching +- `expect.All(comparators ...Comparator)`: allows to bundle together a bunch of other comparators +- `expect.JSON[T any](obj T, verifier func(T, string, tig.T))`: allows to verify the output is valid JSON and optionally +pass `verifier(T, string, tig.T)` extra validation + +### A complete example + +```go +package main + +import ( + "testing" + "errors" + + "github.com/containerd/nerdctl/mod/tigron/tig" + "github.com/containerd/nerdctl/mod/tigron/test" + "github.com/containerd/nerdctl/mod/tigron/expect" +) + +type Thing struct { + Name string +} + +func TestMyThing(t *testing.T) { + // Declare your test + myTest := &test.Case{} + + // Attach a command to run + myTest.Command = test.Custom("bash", "-c", "--", ">&2 echo thing; echo '{\"Name\": \"out\"}'; exit 42;") + + // Set your expectations + myTest.Expected = test.Expects( + expect.ExitCodeGenericFail, + []error{errors.New("thing")}, + expect.All( + expect.Contains("out"), + expect.DoesNotContain("something"), + expect.JSON(&Thing{}, func(obj *Thing, info string, t tig.T) { + assert.Equal(t, obj.Name, "something", info) + }), + ), + ) + + // Run it + myTest.Run(t) +} +``` + +### Custom stdout comparators + +If you need to implement more advanced verifications on stdout that the ready-made comparators can't do, +you can implement your own custom `test.Comparator`. + +For example: + +```go +package whatever + +import ( + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/mod/tigron/tig" + "github.com/containerd/nerdctl/mod/tigron/test" +) + +func TestMyThing(t *testing.T) { + // Declare your test + myTest := &test.Case{} + + // Attach a command to run + myTest.Command = test.Custom("ls") + + // Set your expectations + myTest.Expected = test.Expects(0, nil, func(stdout, info string, t tig.T){ + t.Helper() + // Bla bla, do whatever advanced stuff and some asserts + }) + + // Run it + myTest.Run(t) +} + +// You can of course generalize your comparator into a generator if it is going to be useful repeatedly + +func MyComparatorGenerator(param1, param2 any) test.Comparator { + return func(stdout, info string, t tig.T) { + t.Helper() + // Do your thing... + // ... + } +} + +``` + +You can now pass along `MyComparator(comparisonString)` as the third parameter of `test.Expects`, or compose it with +other comparators using `expect.All(MyComparator(comparisonString), OtherComparator(somethingElse))` + +Note that you have access to an opaque `info` string, that provides a brief formatted header message that assert +will use in case of failure to provide context on the error. +You may of course ignore it and write your own message. + +### Advanced expectations + +You may want to have expectations that contain a certain piece of data that is being used in the command or at +other stages of your test (like `Setup` for example). + +To achieve that, you should write your own `test.Manager` instead of using the helper `test.Expects`. + +A manager is a simple function which only role is to return a `test.Expected` struct. +The `test.Manager` signature makes available `test.Data` and `test.Helpers` to you. + +Here is an example, where we are using `data.Get("sometestdata")`. + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/mod/tigron/test" +) + +func TestMyThing(t *testing.T) { + // Declare your test + myTest := &test.Case{} + + myTest.Setup = func(data test.Data, helpers test.Helpers){ + // Do things... + // ... + // Save this for later + data.Set("something", "lalala") + } + + // Attach a command to run + myTest.Command = test.Custom("somecommand") + + // Set your fully custom expectations + myTest.Expected = func(data test.Data, helpers test.Helpers) *test.Expected { + // With a custom Manager you have access to both the test.Data and test.Helpers to perform more + // refined verifications. + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + }, + Output: func(stdout, info string, t tig.T) { + t.Helper() + + // Retrieve the data that was set during the Setup phase. + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + } + + // Run it + myTest.Run(t) +} +``` diff --git a/mod/tigron/internal/com/command_test.go b/mod/tigron/internal/com/command_test.go index 7652a772675..9c5d0b7e8a0 100644 --- a/mod/tigron/internal/com/command_test.go +++ b/mod/tigron/internal/com/command_test.go @@ -411,7 +411,6 @@ func TestStdoutStderr(t *testing.T) { func TestTimeoutPlain(t *testing.T) { t.Parallel() - start := time.Now() command := &com.Command{ Binary: "bash", // XXX unclear if windows is really able to terminate sleep 5, so, split it up to give it a @@ -421,11 +420,10 @@ func TestTimeoutPlain(t *testing.T) { } err := command.Run(context.WithValue(context.Background(), com.LoggerKey, t)) - assertive.ErrorIsNil(t, err, "Err") + start := time.Now() res, err := command.Wait() - end := time.Now() assertive.ErrorIs(t, err, com.ErrTimeout, "Err") @@ -438,7 +436,6 @@ func TestTimeoutPlain(t *testing.T) { func TestTimeoutDelayed(t *testing.T) { t.Parallel() - start := time.Now() command := &com.Command{ Binary: "bash", // XXX unclear if windows is really able to terminate sleep 5, so, split it up to give it a @@ -448,20 +445,20 @@ func TestTimeoutDelayed(t *testing.T) { } err := command.Run(context.WithValue(context.Background(), com.LoggerKey, t)) - assertive.ErrorIsNil(t, err, "Err") - time.Sleep(1 * time.Second) + start := time.Now() - res, err := command.Wait() + time.Sleep(2 * time.Second) + res, err := command.Wait() end := time.Now() assertive.ErrorIs(t, err, com.ErrTimeout, "Err") assertive.IsEqual(t, res.ExitCode, -1, "ExitCode") assertive.IsEqual(t, res.Stdout, "one", "Stdout") assertive.IsEqual(t, res.Stderr, "", "Stderr") - assertive.IsLessThan(t, end.Sub(start), 2*time.Second, "Total execution time") + assertive.IsLessThan(t, end.Sub(start), 3*time.Second, "Total execution time") } func TestPTYStdout(t *testing.T) {