From e400d03183536016a1be1bafc5f7a3093a4ff77b Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 19 Dec 2022 14:54:59 +0900 Subject: [PATCH] Implement a simple test filter cf. https://github.com/WebAssembly/wasi-testsuite/issues/29 --- README.md | 10 +++++++ examples/skip.json | 5 ++++ test-runner/.pylintrc | 1 + test-runner/tests/test_test_suite_runner.py | 14 +++++++-- test-runner/wasi_test_runner/__main__.py | 15 ++++++++++ test-runner/wasi_test_runner/filters.py | 24 +++++++++++++++ test-runner/wasi_test_runner/harness.py | 4 ++- .../wasi_test_runner/reporters/console.py | 1 + .../wasi_test_runner/test_suite_runner.py | 29 +++++++++++++++++-- 9 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 examples/skip.json create mode 100644 test-runner/wasi_test_runner/filters.py diff --git a/README.md b/README.md index b94bf3a6..57684a57 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,16 @@ python3 test-runner/wasi_test_runner.py -r adapters/wasmtime.sh # path to a runtime adapter ``` +Optionally you can specify test cases to skip with the `--filter` option. + +```bash +python3 test-runner/wasi_test_runner.py \ + -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \ + ./tests/c/testsuite/ \ + --filter examples/skip.json \ + -r adapters/wasmtime.sh # path to a runtime adapter +``` + ## Contributing All contributions are very welcome. Contributors can help with: diff --git a/examples/skip.json b/examples/skip.json new file mode 100644 index 00000000..daa4827b --- /dev/null +++ b/examples/skip.json @@ -0,0 +1,5 @@ +{ + "WASI C tests": { + "stat-dev-ino": "d_ino is known broken in this combination of engine and wasi-sdk version." + } +} diff --git a/test-runner/.pylintrc b/test-runner/.pylintrc index 45c76105..2f210cc1 100644 --- a/test-runner/.pylintrc +++ b/test-runner/.pylintrc @@ -3,5 +3,6 @@ disable= C0114, # Missing module docstring C0115, # Missing class docstring C0116, # Missing function or method docstring + R0903, # Too few public methods [FORMAT] max-line-length=120 diff --git a/test-runner/tests/test_test_suite_runner.py b/test-runner/tests/test_test_suite_runner.py index 75d75c51..d73438de 100644 --- a/test-runner/tests/test_test_suite_runner.py +++ b/test-runner/tests/test_test_suite_runner.py @@ -62,8 +62,12 @@ def test_runner_end_to_end() -> None: reporters = [Mock(), Mock()] + filt = Mock() + filt.should_skip.return_value = (False, None) + filters = [filt] + with patch("glob.glob", return_value=test_paths): - suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters) # type: ignore + suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters) # type: ignore # Assert manifest was read correctly assert suite.name == "test-suite" @@ -91,9 +95,15 @@ def test_runner_end_to_end() -> None: for config, output in zip(expected_config, outputs): validator.assert_any_call(config, output) + # Assert filter calls + for filt in filters: + assert filt.should_skip.call_count == 3 + for test_case in expected_test_cases: + filt.should_skip.assert_any_call(suite.name, test_case.name) + @patch("os.path.exists", Mock(return_value=False)) def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None: - suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], []) + suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], []) assert suite.name == "my-path" diff --git a/test-runner/wasi_test_runner/__main__.py b/test-runner/wasi_test_runner/__main__.py index 1eef0e1f..8f033ab5 100644 --- a/test-runner/wasi_test_runner/__main__.py +++ b/test-runner/wasi_test_runner/__main__.py @@ -5,6 +5,8 @@ from .runtime_adapter import RuntimeAdapter from .harness import run_all_tests +from .filters import TestFilter +from .filters import JSONTestFilter from .reporters import TestReporter from .reporters.console import ConsoleTestReporter from .reporters.json import JSONTestReporter @@ -23,6 +25,14 @@ def main() -> int: nargs="+", help="Locations of suites (directories with *.wasm test files).", ) + parser.add_argument( + "-f", + "--filter", + required=False, + nargs="+", + default=[], + help="Locations of test filters (JSON files).", + ) parser.add_argument( "-r", "--runtime-adapter", required=True, help="Path to a runtime adapter." ) @@ -45,11 +55,16 @@ def main() -> int: validators: List[Validator] = [exit_code_validator, stdout_validator] + filters: List[TestFilter] = [] + for filt in options.filter: + filters.append(JSONTestFilter(filt)) + return run_all_tests( RuntimeAdapter(options.runtime_adapter), options.test_suite, validators, reporters, + filters, ) diff --git a/test-runner/wasi_test_runner/filters.py b/test-runner/wasi_test_runner/filters.py new file mode 100644 index 00000000..f692557a --- /dev/null +++ b/test-runner/wasi_test_runner/filters.py @@ -0,0 +1,24 @@ +from typing import Tuple, Any +from abc import ABC +from abc import abstractmethod + +import json + + +class TestFilter(ABC): + @abstractmethod + def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]: + pass + + +class JSONTestFilter(TestFilter): + def __init__(self, filename: str) -> None: + with open(filename, encoding="utf-8") as file: + self.filter_dict = json.load(file) + + def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]: + test_suite_filter = self.filter_dict.get(test_suite_name) + if test_suite_filter is None: + return False, None + why = test_suite_filter.get(test_name) + return why is not None, why diff --git a/test-runner/wasi_test_runner/harness.py b/test-runner/wasi_test_runner/harness.py index d440b30a..027b2bb5 100644 --- a/test-runner/wasi_test_runner/harness.py +++ b/test-runner/wasi_test_runner/harness.py @@ -1,5 +1,6 @@ from typing import List +from .filters import TestFilter from .reporters import TestReporter from .test_suite_runner import run_tests_from_test_suite from .runtime_adapter import RuntimeAdapter @@ -11,12 +12,13 @@ def run_all_tests( test_suite_paths: List[str], validators: List[Validator], reporters: List[TestReporter], + filters: List[TestFilter], ) -> int: ret = 0 for test_suite_path in test_suite_paths: test_suite = run_tests_from_test_suite( - test_suite_path, runtime, validators, reporters + test_suite_path, runtime, validators, reporters, filters, ) for reporter in reporters: reporter.report_test_suite(test_suite) diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index 9bbae086..3367cf9d 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -54,6 +54,7 @@ def finalize(self, version: RuntimeVersion) -> None: print(f" Total: {suite.test_count}") self._print_pass(f" Passed: {suite.pass_count}") self._print_fail(f" Failed: {suite.fail_count}") + self._print_skip(f" Skipped: {suite.skip_count}") print("") print( diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index 47a83c9e..c9a26b14 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -8,6 +8,7 @@ from datetime import datetime from typing import List, cast +from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( Result, @@ -25,14 +26,26 @@ def run_tests_from_test_suite( runtime: RuntimeAdapter, validators: List[Validator], reporters: List[TestReporter], + filters: List[TestFilter], ) -> TestSuite: test_cases: List[TestCase] = [] test_start = datetime.now() _cleanup_test_output(test_suite_path) + test_suite_name = _read_manifest(test_suite_path) + for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")): - test_case = _execute_single_test(runtime, validators, test_path) + test_name = os.path.splitext(os.path.basename(test_path))[0] + for filt in filters: + # for now, just drop the skip reason string. it might be + # useful to make reporters report it. + skip, _ = filt.should_skip(test_suite_name, test_name) + if skip: + test_case = _skip_single_test(runtime, validators, test_path) + break + else: + test_case = _execute_single_test(runtime, validators, test_path) test_cases.append(test_case) for reporter in reporters: reporter.report_test(test_case) @@ -40,13 +53,25 @@ def run_tests_from_test_suite( elapsed = (datetime.now() - test_start).total_seconds() return TestSuite( - name=_read_manifest(test_suite_path), + name=test_suite_name, time=test_start, duration_s=elapsed, test_cases=test_cases, ) +def _skip_single_test( + _runtime: RuntimeAdapter, _validators: List[Validator], test_path: str +) -> TestCase: + config = _read_test_config(test_path) + return TestCase( + name=os.path.splitext(os.path.basename(test_path))[0], + config=config, + result=Result(output=Output(0, "", ""), is_executed=False, failures=[]), + duration_s=0, + ) + + def _execute_single_test( runtime: RuntimeAdapter, validators: List[Validator], test_path: str ) -> TestCase: