From 1120cf7cfd3c3d17700edfe51779b7b430eeecda Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 31 Jan 2023 17:56:49 +0900 Subject: [PATCH 1/4] Refactor Result class I plan to add "timed out" case to the union later. --- .../wasi_test_runner/reporters/console.py | 10 ++++----- test-runner/wasi_test_runner/test_case.py | 14 +++++++------ test-runner/wasi_test_runner/test_suite.py | 8 +++---- .../wasi_test_runner/test_suite_runner.py | 21 +++++++++++-------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index ff8ab585e..58a9fdb07 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -2,7 +2,7 @@ from colorama import Fore, init from . import TestReporter -from ..test_case import TestCase +from ..test_case import TestCase, Skipped from ..test_suite import TestSuite from ..runtime_adapter import RuntimeVersion @@ -19,7 +19,9 @@ def __init__(self, colored: bool = True) -> None: self._colored = colored def report_test(self, test: TestCase) -> None: - if test.result.failed: + if isinstance(test.result, Skipped): + self._print_skip(f"Test {test.name} skipped: {test.result.reason}") + elif test.result.failures: self._print_fail(f"Test {test.name} failed") for reason in test.result.failures: self._print_fail(f" [{reason.type}] {reason.message}") @@ -27,10 +29,8 @@ def report_test(self, test: TestCase) -> None: print(test.result.output.stdout) print("STDERR:") print(test.result.output.stderr) - elif test.result.is_executed: - self._print_pass(f"Test {test.name} passed") else: - self._print_skip(f"Test {test.name} skipped") + self._print_pass(f"Test {test.name} passed") def report_test_suite(self, test_suite: TestSuite) -> None: self._test_suites.append(test_suite) diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eebb95d3f..ad950eff0 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -1,6 +1,6 @@ import logging import json -from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional +from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional, Union class Output(NamedTuple): @@ -14,14 +14,16 @@ class Failure(NamedTuple): message: str -class Result(NamedTuple): +class Executed(NamedTuple): output: Output - is_executed: bool failures: List[Failure] - @property - def failed(self) -> bool: - return len(self.failures) > 0 + +class Skipped(NamedTuple): + reason: str + + +Result = Union[Executed, Skipped] T = TypeVar("T", bound="Config") diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py index 4681cd9a6..5b4338620 100644 --- a/test-runner/wasi_test_runner/test_suite.py +++ b/test-runner/wasi_test_runner/test_suite.py @@ -1,6 +1,6 @@ from typing import NamedTuple, List from datetime import datetime -from .test_case import TestCase +from .test_case import TestCase, Executed, Skipped class TestSuite(NamedTuple): @@ -19,7 +19,7 @@ def pass_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed is False + if isinstance(test.result, Executed) and not test.result.failures ] ) @@ -29,10 +29,10 @@ def fail_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed + if isinstance(test.result, Executed) and test.result.failures ] ) @property def skip_count(self) -> int: - return len([1 for test in self.test_cases if not test.result.is_executed]) + return len([1 for test in self.test_cases if isinstance(test.result, Skipped)]) diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index c9a26b149..f6bde9509 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -11,7 +11,8 @@ from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( - Result, + Executed, + Skipped, Config, Output, TestCase, @@ -38,11 +39,13 @@ def run_tests_from_test_suite( for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")): test_name = os.path.splitext(os.path.basename(test_path))[0] for filt in filters: - # for now, just drop the skip reason string. it might be - # useful to make reporters report it. - skip, _ = filt.should_skip(test_suite_name, test_name) + skip, reason = filt.should_skip(test_suite_name, test_name) if skip: - test_case = _skip_single_test(runtime, validators, test_path) + # Given the type of should_skip, "if skip" implies reason + # is str. But mypy doesn't seem to recognize it. Use an + # explicit cast for now. + reason = cast(str, reason) + test_case = _skip_single_test(runtime, validators, test_path, reason) break else: test_case = _execute_single_test(runtime, validators, test_path) @@ -61,13 +64,13 @@ def run_tests_from_test_suite( def _skip_single_test( - _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str + _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str, reason: str ) -> TestCase: config = _read_test_config(test_path) return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=Result(output=Output(0, "", ""), is_executed=False, failures=[]), + result=Skipped(reason), duration_s=0, ) @@ -88,14 +91,14 @@ def _execute_single_test( ) -def _validate(validators: List[Validator], config: Config, output: Output) -> Result: +def _validate(validators: List[Validator], config: Config, output: Output) -> Executed: failures = [ result for result in [validator(config, output) for validator in validators] if result is not None ] - return Result(failures=failures, is_executed=True, output=output) + return Executed(failures=failures, output=output) def _read_test_config(test_path: str) -> Config: From 3896f369ccfbea13c64ac426d25faa8d09589867 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 21 Feb 2023 19:01:02 +0900 Subject: [PATCH 2/4] rename a few things as suggested Executed -> ExecutedResult Skipped -> SkippedResult --- test-runner/wasi_test_runner/test_case.py | 6 +++--- test-runner/wasi_test_runner/test_suite.py | 8 ++++---- test-runner/wasi_test_runner/test_suite_runner.py | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index ad950eff0..eed09c291 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -14,16 +14,16 @@ class Failure(NamedTuple): message: str -class Executed(NamedTuple): +class ExecutedResult(NamedTuple): output: Output failures: List[Failure] -class Skipped(NamedTuple): +class SkippedResult(NamedTuple): reason: str -Result = Union[Executed, Skipped] +Result = Union[ExecutedResult, SkippedResult] T = TypeVar("T", bound="Config") diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py index 5b4338620..972de44a6 100644 --- a/test-runner/wasi_test_runner/test_suite.py +++ b/test-runner/wasi_test_runner/test_suite.py @@ -1,6 +1,6 @@ from typing import NamedTuple, List from datetime import datetime -from .test_case import TestCase, Executed, Skipped +from .test_case import TestCase, ExecutedResult, SkippedResult class TestSuite(NamedTuple): @@ -19,7 +19,7 @@ def pass_count(self) -> int: [ 1 for test in self.test_cases - if isinstance(test.result, Executed) and not test.result.failures + if isinstance(test.result, ExecutedResult) and not test.result.failures ] ) @@ -29,10 +29,10 @@ def fail_count(self) -> int: [ 1 for test in self.test_cases - if isinstance(test.result, Executed) and test.result.failures + if isinstance(test.result, ExecutedResult) and test.result.failures ] ) @property def skip_count(self) -> int: - return len([1 for test in self.test_cases if isinstance(test.result, Skipped)]) + return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)]) diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index f6bde9509..7f00aa342 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -11,8 +11,8 @@ from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( - Executed, - Skipped, + ExecutedResult, + SkippedResult, Config, Output, TestCase, @@ -70,7 +70,7 @@ def _skip_single_test( return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=Skipped(reason), + result=SkippedResult(reason), duration_s=0, ) @@ -91,14 +91,14 @@ def _execute_single_test( ) -def _validate(validators: List[Validator], config: Config, output: Output) -> Executed: +def _validate(validators: List[Validator], config: Config, output: Output) -> ExecutedResult: failures = [ result for result in [validator(config, output) for validator in validators] if result is not None ] - return Executed(failures=failures, output=output) + return ExecutedResult(failures=failures, output=output) def _read_test_config(test_path: str) -> Config: From de2b178efe9eda17885a544d993e5e1aae34afa7 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 21 Feb 2023 19:03:42 +0900 Subject: [PATCH 3/4] restore failed property as suggested --- test-runner/wasi_test_runner/test_case.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eed09c291..79b960848 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -18,6 +18,10 @@ class ExecutedResult(NamedTuple): output: Output failures: List[Failure] + @property + def failed(self) -> bool: + return len(self.failures) > 0 + class SkippedResult(NamedTuple): reason: str From f7941e71a3904aa80e80bdd79363e05c81916c03 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 21 Feb 2023 19:06:05 +0900 Subject: [PATCH 4/4] restore failed property as suggested --- test-runner/wasi_test_runner/reporters/console.py | 2 +- test-runner/wasi_test_runner/test_suite.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index 58a9fdb07..802259a6a 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -21,7 +21,7 @@ def __init__(self, colored: bool = True) -> None: def report_test(self, test: TestCase) -> None: if isinstance(test.result, Skipped): self._print_skip(f"Test {test.name} skipped: {test.result.reason}") - elif test.result.failures: + elif test.result.failed: self._print_fail(f"Test {test.name} failed") for reason in test.result.failures: self._print_fail(f" [{reason.type}] {reason.message}") diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py index 972de44a6..aab8f3094 100644 --- a/test-runner/wasi_test_runner/test_suite.py +++ b/test-runner/wasi_test_runner/test_suite.py @@ -19,7 +19,7 @@ def pass_count(self) -> int: [ 1 for test in self.test_cases - if isinstance(test.result, ExecutedResult) and not test.result.failures + if isinstance(test.result, ExecutedResult) and not test.result.failed ] ) @@ -29,7 +29,7 @@ def fail_count(self) -> int: [ 1 for test in self.test_cases - if isinstance(test.result, ExecutedResult) and test.result.failures + if isinstance(test.result, ExecutedResult) and test.result.failed ] )