From e1a925ff832b617579f16b6a86f0a0a707636bdf Mon Sep 17 00:00:00 2001 From: Joel Spadin Date: Mon, 16 Jan 2023 12:06:43 -0600 Subject: [PATCH] refactor(test): Run tests with pytest Replaced the run-test.sh script with a Python script that runs our unit tests through pytest. Tests are now run in parallel to speed up running the entire test suite, and it allows for integration with other tools that support pytest, such as IDEs. Also removed a dependency on remarshal, because it depends on an old version of PyYAML that conflicts with other Python packages in our Docker image. Replaced it with yq. --- .github/workflows/build-user-config.yml | 12 +- .github/workflows/test.yml | 5 +- .vscode/settings.json | 6 +- app/run-test.sh | 47 ------ app/scripts/requirements.txt | 10 +- app/scripts/west_commands/test.py | 26 ++-- app/test_zmk.py | 187 ++++++++++++++++++++++++ docs/docs/development/tests.md | 30 +++- 8 files changed, 247 insertions(+), 76 deletions(-) delete mode 100755 app/run-test.sh create mode 100644 app/test_zmk.py diff --git a/.github/workflows/build-user-config.yml b/.github/workflows/build-user-config.yml index 3d89ed7b..323b8d4a 100644 --- a/.github/workflows/build-user-config.yml +++ b/.github/workflows/build-user-config.yml @@ -29,18 +29,16 @@ jobs: runs-on: ubuntu-latest name: Fetch Build Keyboards outputs: - build_matrix: ${{ env.build_matrix }} + build_matrix: ${{ steps.fetch_build_matrix.outputs.result }} steps: - name: Checkout uses: actions/checkout@v3 - - name: Install yaml2json - run: python3 -m pip install remarshal - - name: Fetch Build Matrix - run: | - echo "build_matrix=$(yaml2json ${{ inputs.build_matrix_path }} | jq -c .)" >> $GITHUB_ENV - yaml2json ${{ inputs.build_matrix_path }} | jq + id: fetch_build_matrix + uses: mikefarah/yq@v4.30.8 + with: + cmd: yq -o json ${{ inputs.build_matrix_path }} build: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38c61eea..ce89027e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,10 +33,11 @@ jobs: test: ${{ fromJSON(needs.collect-tests.outputs.test-dirs) }} runs-on: ubuntu-latest container: - image: docker.io/zmkfirmware/zmk-build-arm:3.0 + image: docker.io/zmkfirmware/zmk-build-arm:stable steps: - name: Checkout uses: actions/checkout@v3 + - name: Cache west modules uses: actions/cache@v3.0.2 env: @@ -62,7 +63,7 @@ jobs: run: west zephyr-export - name: Test ${{ matrix.test }} working-directory: app - run: west test tests/${{ matrix.test }} + run: west test ${{ matrix.test }} - name: Archive artifacts if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.vscode/settings.json b/.vscode/settings.json index 2730549a..28d4dffb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,9 @@ "*.overlay": "dts", "*.keymap": "dts" }, - "python.formatting.provider": "black" + "python.formatting.provider": "black", + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.cwd": "${workspaceFolder}/app", + "python.testing.pytestArgs": ["test_zmk.py", "-v", "--numprocesses=auto"] } \ No newline at end of file diff --git a/app/run-test.sh b/app/run-test.sh deleted file mode 100755 index 068fdbb4..00000000 --- a/app/run-test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2020 The ZMK Contributors -# SPDX-License-Identifier: MIT - -if [ -z "$1" ]; then - echo "Usage: ./run-test.sh " - exit 1 -fi - -path="$1" -if [ $path = "all" ]; then - path="tests" -fi - -testcases=$(find $path -name native_posix_64.keymap -exec dirname \{\} \;) -num_cases=$(echo "$testcases" | wc -l) -if [ $num_cases -gt 1 ] || [ "$testcases" != "$path" ]; then - echo "" > ./build/tests/pass-fail.log - echo "$testcases" | xargs -L 1 -P ${J:-4} ./run-test.sh - err=$? - sort -k2 ./build/tests/pass-fail.log - exit $err -fi - -testcase="$path" -echo "Running $testcase:" - -west build -d build/$testcase -b native_posix_64 -- -DZMK_CONFIG="$(pwd)/$testcase" > /dev/null 2>&1 -if [ $? -gt 0 ]; then - echo "FAILED: $testcase did not build" | tee -a ./build/tests/pass-fail.log - exit 1 -fi - -./build/$testcase/zephyr/zmk.exe | sed -e "s/.*> //" | tee build/$testcase/keycode_events_full.log | sed -n -f $testcase/events.patterns > build/$testcase/keycode_events.log -diff -au $testcase/keycode_events.snapshot build/$testcase/keycode_events.log -if [ $? -gt 0 ]; then - if [ -f $testcase/pending ]; then - echo "PENDING: $testcase" | tee -a ./build/tests/pass-fail.log - exit 0 - fi - echo "FAILED: $testcase" | tee -a ./build/tests/pass-fail.log - exit 1 -fi - -echo "PASS: $testcase" | tee -a ./build/tests/pass-fail.log -exit 0 diff --git a/app/scripts/requirements.txt b/app/scripts/requirements.txt index 60d6f3ae..043c9162 100644 --- a/app/scripts/requirements.txt +++ b/app/scripts/requirements.txt @@ -1,8 +1,10 @@ # Copyright (c) 2021 The ZMK Contributors # SPDX-License-Identifier: MIT -# Convert YAML to JSON for validation -remarshal>=0.14.0 - # Perform our hardware metadata validation -jsonschema>=3.2.0 \ No newline at end of file +jsonschema>=3.2.0 + +# Unit testing +pythonsed>=2.1 +pytest>=7.0.0 +pytest-xdist>=3.0.0 diff --git a/app/scripts/west_commands/test.py b/app/scripts/west_commands/test.py index 53133491..890502ee 100644 --- a/app/scripts/west_commands/test.py +++ b/app/scripts/west_commands/test.py @@ -2,9 +2,7 @@ # SPDX-License-Identifier: MIT """Test runner for ZMK.""" -import os -import subprocess - +import pytest from west.commands import WestCommand from west import log # use this for user output @@ -14,7 +12,8 @@ class Test(WestCommand): super().__init__( name="test", help="run ZMK testsuite", - description="Run the ZMK testsuite.", + description="Run the ZMK testsuite. Arguments are passed through to pytest.", + accepts_unknown_args=True, ) def do_add_parser(self, parser_adder): @@ -25,17 +24,18 @@ class Test(WestCommand): ) parser.add_argument( - "test_path", - default="all", - help='The path to the test. Defaults to "all".', + "test", + help="The path to the test to run. Runs all tests if not specified.", nargs="?", ) return parser def do_run(self, args, unknown_args): - # the run-test script assumes the app directory is the current dir. - os.chdir(f"{self.topdir}/app") - completed_process = subprocess.run( - [f"{self.topdir}/app/run-test.sh", args.test_path] - ) - exit(completed_process.returncode) + pytest_args = [f"{self.topdir}/app/test_zmk.py", "--numprocesses=auto"] + pytest_args += unknown_args + + if args.test: + pytest_args += ["-k", args.test] + + returncode = pytest.main(pytest_args) + exit(returncode) diff --git a/app/test_zmk.py b/app/test_zmk.py new file mode 100644 index 00000000..dbb0f911 --- /dev/null +++ b/app/test_zmk.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 The ZMK Contributors +# SPDX-License-Identifier: MIT + + +from io import StringIO +from pathlib import Path +import subprocess +from subprocess import CalledProcessError, PIPE +from typing import Sequence, Union + +from PythonSed import Sed +import pytest + + +APP_PATH = Path(__file__).parent +TESTS_PATH = APP_PATH / "tests" +BUILD_PATH = APP_PATH / "build/tests" + +# TODO: change all tests to build with a unit test shield +TEST_KEYMAP_NAME = "native_posix_64.keymap" + + +def check_call_teed(args: Sequence[Union[str, Path]], **kwargs): + """ + Call a subprocess, print its output, and also return its output. + """ + with subprocess.Popen( + args, bufsize=1, stdout=PIPE, text=True, encoding="utf-8", **kwargs + ) as p: + stdout = StringIO() + + for line in p.stdout: + stdout.write(line) + print(line, end="") + + returncode = p.wait() + output = stdout.getvalue() + + if returncode: + raise CalledProcessError(returncode, p.args, output) + + return output + + +def strip_log_tags(log: str): + def strip(line: str): + left, _, right = line.partition("> ") + return right or left + + return "\n".join(strip(line) for line in log.splitlines()) + + +def filter_log(log: str, patterns_file: Path): + sed = Sed(encoding="utf-8") + sed.no_autoprint = True + sed.load_script(str(patterns_file)) + + input = StringIO(log) + output = StringIO() + sed.apply(input, output) + + return output.getvalue() + + +class Runner: + def __init__(self, path: Path, build_path: Path): + self.path = path + self.build_path = build_path + + @property + def board(self) -> str: + """The Zephyr board to build""" + raise NotImplementedError() + + @classmethod + def check(cls): + """Check that the environment is valid for the runner.""" + pass + + def run(self) -> str: + """Run the unit test and return its output.""" + raise NotImplementedError() + + +class PosixRunner(Runner): + @property + def board(self): + return "native_posix_64" + + def run(self): + return check_call_teed([self.build_path / "zephyr/zmk.exe"], cwd=APP_PATH) + + +# TODO: Add QemuRunner for non-Posix platforms + + +class ZmkTestCase: + def __init__(self, path: Path) -> None: + self.path = path + self.runner = self._get_runner() + self.runner.check() + + self.patterns_file = self.path / "events.patterns" + self.snapshot_file = self.path / "keycode_events.snapshot" + + if not self.patterns_file.exists(): + pytest.fail(f"Missing patterns file: {self.patterns_file}") + + if not self.snapshot_file.exists(): + pytest.fail(f"Missing snapshot file: {self.snapshot_file}") + + def run(self): + if reason := self.get_pending_reason(): + pytest.skip(reason) + + self._build() + self._test() + + @property + def build_path(self): + return BUILD_PATH / self.path.relative_to(TESTS_PATH) + + @property + def relative_build_path(self): + return self.build_path.relative_to(APP_PATH) + + def get_pending_reason(self): + for file in ["pending", f"pending-{self.runner.board}"]: + try: + return (self.path / file).read_text(encoding="utf-8") + except: + pass + + return None + + def _build(self): + subprocess.check_call( + [ + "west", + "build", + "-d", + self.build_path.relative_to(APP_PATH).as_posix(), + "-b", + self.runner.board, + "--", + f"-DZMK_CONFIG={self.path.as_posix()}", + ], + cwd=APP_PATH, + ) + + def _test(self): + output = self.runner.run() + output = strip_log_tags(output) + with self._open_log("keycode_events_full.log") as log: + log.write(output) + + output = filter_log(output, self.patterns_file) + with self._open_log("keycode_events.log") as log: + log.write(output) + + assert output == self.snapshot_file.read_text(encoding="utf-8") + + def _get_runner(self) -> type[Runner]: + # TODO: return a QemuRunner for non-Posix platforms + return PosixRunner(self.path, self.build_path) + + def _open_log(self, name: str, mode="w"): + log_path = self.build_path / name + log_path.parent.mkdir(parents=True, exist_ok=True) + + return log_path.open(mode, encoding="utf-8") + + +def get_tests(): + paths = sorted(keymap.parent for keymap in TESTS_PATH.rglob(TEST_KEYMAP_NAME)) + + return [ + pytest.param(path, id=str(path.relative_to(TESTS_PATH).as_posix())) + for path in paths + ] + + +@pytest.mark.parametrize("name", get_tests()) +def test(name): + ZmkTestCase(TESTS_PATH / name).run() diff --git a/docs/docs/development/tests.md b/docs/docs/development/tests.md index 37b52bdd..480fb0e2 100644 --- a/docs/docs/development/tests.md +++ b/docs/docs/development/tests.md @@ -5,8 +5,34 @@ sidebar_label: Tests - Running tests requires [native posix support](posix-board.md). - Any folder under `/app/tests` containing `native_posix_64.keymap` will be selected when running `west test`. -- Run tests from within the `/zmk/app` directory. -- Run a single test with `west test `, like `west test tests/toggle-layer/normal`. + +## Running Tests + +All the following commands assume you have a terminal opened to the `zmk/app` directory. + +First, make sure all Python dependencies are installed: + +```sh +python3 -m pip install -r scripts/requirements.txt +``` + +Tests can then be run from Zephyr's [west](https://docs.zephyrproject.org/3.2.0/develop/west/index.html) tool with the `test` subcommand: + +```sh +west test +``` + +Running it with no arguments will run the entire test suite. You can run a single test case or test set by providing the relative path from `zmk/app/tests` to the test directory, e.g. + +```sh +# Run all tests cases in the toggle-layer set +west test toggle-layer + +# Run the toggle-layer/normal test case +west test toggle-layer/normal +``` + +Any additional arguments are passed through to [PyTest](https://docs.pytest.org/). ## Creating a New Test Set