diff --git a/.github/workflows/build-user-config.yml b/.github/workflows/build-user-config.yml index 3d89ed7b..323b8d4a 100644 --- a/.github/workflows/build-user-config.yml +++ b/.github/workflows/build-user-config.yml @@ -29,18 +29,16 @@ jobs: runs-on: ubuntu-latest name: Fetch Build Keyboards outputs: - build_matrix: ${{ env.build_matrix }} + build_matrix: ${{ steps.fetch_build_matrix.outputs.result }} steps: - name: Checkout uses: actions/checkout@v3 - - name: Install yaml2json - run: python3 -m pip install remarshal - - name: Fetch Build Matrix - run: | - echo "build_matrix=$(yaml2json ${{ inputs.build_matrix_path }} | jq -c .)" >> $GITHUB_ENV - yaml2json ${{ inputs.build_matrix_path }} | jq + id: fetch_build_matrix + uses: mikefarah/yq@v4.30.8 + with: + cmd: yq -o json ${{ inputs.build_matrix_path }} build: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38c61eea..ce89027e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,10 +33,11 @@ jobs: test: ${{ fromJSON(needs.collect-tests.outputs.test-dirs) }} runs-on: ubuntu-latest container: - image: docker.io/zmkfirmware/zmk-build-arm:3.0 + image: docker.io/zmkfirmware/zmk-build-arm:stable steps: - name: Checkout uses: actions/checkout@v3 + - name: Cache west modules uses: actions/cache@v3.0.2 env: @@ -62,7 +63,7 @@ jobs: run: west zephyr-export - name: Test ${{ matrix.test }} working-directory: app - run: west test tests/${{ matrix.test }} + run: west test ${{ matrix.test }} - name: Archive artifacts if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.vscode/settings.json b/.vscode/settings.json index 2730549a..28d4dffb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,9 @@ "*.overlay": "dts", "*.keymap": "dts" }, - "python.formatting.provider": "black" + "python.formatting.provider": "black", + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.cwd": "${workspaceFolder}/app", + "python.testing.pytestArgs": ["test_zmk.py", "-v", "--numprocesses=auto"] } \ No newline at end of file diff --git a/app/run-test.sh b/app/run-test.sh deleted file mode 100755 index 068fdbb4..00000000 --- a/app/run-test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2020 The ZMK Contributors -# SPDX-License-Identifier: MIT - -if [ -z "$1" ]; then - echo "Usage: ./run-test.sh " - exit 1 -fi - -path="$1" -if [ $path = "all" ]; then - path="tests" -fi - -testcases=$(find $path -name native_posix_64.keymap -exec dirname \{\} \;) -num_cases=$(echo "$testcases" | wc -l) -if [ $num_cases -gt 1 ] || [ "$testcases" != "$path" ]; then - echo "" > ./build/tests/pass-fail.log - echo "$testcases" | xargs -L 1 -P ${J:-4} ./run-test.sh - err=$? - sort -k2 ./build/tests/pass-fail.log - exit $err -fi - -testcase="$path" -echo "Running $testcase:" - -west build -d build/$testcase -b native_posix_64 -- -DZMK_CONFIG="$(pwd)/$testcase" > /dev/null 2>&1 -if [ $? -gt 0 ]; then - echo "FAILED: $testcase did not build" | tee -a ./build/tests/pass-fail.log - exit 1 -fi - -./build/$testcase/zephyr/zmk.exe | sed -e "s/.*> //" | tee build/$testcase/keycode_events_full.log | sed -n -f $testcase/events.patterns > build/$testcase/keycode_events.log -diff -au $testcase/keycode_events.snapshot build/$testcase/keycode_events.log -if [ $? -gt 0 ]; then - if [ -f $testcase/pending ]; then - echo "PENDING: $testcase" | tee -a ./build/tests/pass-fail.log - exit 0 - fi - echo "FAILED: $testcase" | tee -a ./build/tests/pass-fail.log - exit 1 -fi - -echo "PASS: $testcase" | tee -a ./build/tests/pass-fail.log -exit 0 diff --git a/app/scripts/requirements.txt b/app/scripts/requirements.txt index 60d6f3ae..043c9162 100644 --- a/app/scripts/requirements.txt +++ b/app/scripts/requirements.txt @@ -1,8 +1,10 @@ # Copyright (c) 2021 The ZMK Contributors # SPDX-License-Identifier: MIT -# Convert YAML to JSON for validation -remarshal>=0.14.0 - # Perform our hardware metadata validation -jsonschema>=3.2.0 \ No newline at end of file +jsonschema>=3.2.0 + +# Unit testing +pythonsed>=2.1 +pytest>=7.0.0 +pytest-xdist>=3.0.0 diff --git a/app/scripts/west_commands/test.py b/app/scripts/west_commands/test.py index 53133491..890502ee 100644 --- a/app/scripts/west_commands/test.py +++ b/app/scripts/west_commands/test.py @@ -2,9 +2,7 @@ # SPDX-License-Identifier: MIT """Test runner for ZMK.""" -import os -import subprocess - +import pytest from west.commands import WestCommand from west import log # use this for user output @@ -14,7 +12,8 @@ class Test(WestCommand): super().__init__( name="test", help="run ZMK testsuite", - description="Run the ZMK testsuite.", + description="Run the ZMK testsuite. Arguments are passed through to pytest.", + accepts_unknown_args=True, ) def do_add_parser(self, parser_adder): @@ -25,17 +24,18 @@ class Test(WestCommand): ) parser.add_argument( - "test_path", - default="all", - help='The path to the test. Defaults to "all".', + "test", + help="The path to the test to run. Runs all tests if not specified.", nargs="?", ) return parser def do_run(self, args, unknown_args): - # the run-test script assumes the app directory is the current dir. - os.chdir(f"{self.topdir}/app") - completed_process = subprocess.run( - [f"{self.topdir}/app/run-test.sh", args.test_path] - ) - exit(completed_process.returncode) + pytest_args = [f"{self.topdir}/app/test_zmk.py", "--numprocesses=auto"] + pytest_args += unknown_args + + if args.test: + pytest_args += ["-k", args.test] + + returncode = pytest.main(pytest_args) + exit(returncode) diff --git a/app/test_zmk.py b/app/test_zmk.py new file mode 100644 index 00000000..dbb0f911 --- /dev/null +++ b/app/test_zmk.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 The ZMK Contributors +# SPDX-License-Identifier: MIT + + +from io import StringIO +from pathlib import Path +import subprocess +from subprocess import CalledProcessError, PIPE +from typing import Sequence, Union + +from PythonSed import Sed +import pytest + + +APP_PATH = Path(__file__).parent +TESTS_PATH = APP_PATH / "tests" +BUILD_PATH = APP_PATH / "build/tests" + +# TODO: change all tests to build with a unit test shield +TEST_KEYMAP_NAME = "native_posix_64.keymap" + + +def check_call_teed(args: Sequence[Union[str, Path]], **kwargs): + """ + Call a subprocess, print its output, and also return its output. + """ + with subprocess.Popen( + args, bufsize=1, stdout=PIPE, text=True, encoding="utf-8", **kwargs + ) as p: + stdout = StringIO() + + for line in p.stdout: + stdout.write(line) + print(line, end="") + + returncode = p.wait() + output = stdout.getvalue() + + if returncode: + raise CalledProcessError(returncode, p.args, output) + + return output + + +def strip_log_tags(log: str): + def strip(line: str): + left, _, right = line.partition("> ") + return right or left + + return "\n".join(strip(line) for line in log.splitlines()) + + +def filter_log(log: str, patterns_file: Path): + sed = Sed(encoding="utf-8") + sed.no_autoprint = True + sed.load_script(str(patterns_file)) + + input = StringIO(log) + output = StringIO() + sed.apply(input, output) + + return output.getvalue() + + +class Runner: + def __init__(self, path: Path, build_path: Path): + self.path = path + self.build_path = build_path + + @property + def board(self) -> str: + """The Zephyr board to build""" + raise NotImplementedError() + + @classmethod + def check(cls): + """Check that the environment is valid for the runner.""" + pass + + def run(self) -> str: + """Run the unit test and return its output.""" + raise NotImplementedError() + + +class PosixRunner(Runner): + @property + def board(self): + return "native_posix_64" + + def run(self): + return check_call_teed([self.build_path / "zephyr/zmk.exe"], cwd=APP_PATH) + + +# TODO: Add QemuRunner for non-Posix platforms + + +class ZmkTestCase: + def __init__(self, path: Path) -> None: + self.path = path + self.runner = self._get_runner() + self.runner.check() + + self.patterns_file = self.path / "events.patterns" + self.snapshot_file = self.path / "keycode_events.snapshot" + + if not self.patterns_file.exists(): + pytest.fail(f"Missing patterns file: {self.patterns_file}") + + if not self.snapshot_file.exists(): + pytest.fail(f"Missing snapshot file: {self.snapshot_file}") + + def run(self): + if reason := self.get_pending_reason(): + pytest.skip(reason) + + self._build() + self._test() + + @property + def build_path(self): + return BUILD_PATH / self.path.relative_to(TESTS_PATH) + + @property + def relative_build_path(self): + return self.build_path.relative_to(APP_PATH) + + def get_pending_reason(self): + for file in ["pending", f"pending-{self.runner.board}"]: + try: + return (self.path / file).read_text(encoding="utf-8") + except: + pass + + return None + + def _build(self): + subprocess.check_call( + [ + "west", + "build", + "-d", + self.build_path.relative_to(APP_PATH).as_posix(), + "-b", + self.runner.board, + "--", + f"-DZMK_CONFIG={self.path.as_posix()}", + ], + cwd=APP_PATH, + ) + + def _test(self): + output = self.runner.run() + output = strip_log_tags(output) + with self._open_log("keycode_events_full.log") as log: + log.write(output) + + output = filter_log(output, self.patterns_file) + with self._open_log("keycode_events.log") as log: + log.write(output) + + assert output == self.snapshot_file.read_text(encoding="utf-8") + + def _get_runner(self) -> type[Runner]: + # TODO: return a QemuRunner for non-Posix platforms + return PosixRunner(self.path, self.build_path) + + def _open_log(self, name: str, mode="w"): + log_path = self.build_path / name + log_path.parent.mkdir(parents=True, exist_ok=True) + + return log_path.open(mode, encoding="utf-8") + + +def get_tests(): + paths = sorted(keymap.parent for keymap in TESTS_PATH.rglob(TEST_KEYMAP_NAME)) + + return [ + pytest.param(path, id=str(path.relative_to(TESTS_PATH).as_posix())) + for path in paths + ] + + +@pytest.mark.parametrize("name", get_tests()) +def test(name): + ZmkTestCase(TESTS_PATH / name).run() diff --git a/docs/docs/development/tests.md b/docs/docs/development/tests.md index 37b52bdd..480fb0e2 100644 --- a/docs/docs/development/tests.md +++ b/docs/docs/development/tests.md @@ -5,8 +5,34 @@ sidebar_label: Tests - Running tests requires [native posix support](posix-board.md). - Any folder under `/app/tests` containing `native_posix_64.keymap` will be selected when running `west test`. -- Run tests from within the `/zmk/app` directory. -- Run a single test with `west test `, like `west test tests/toggle-layer/normal`. + +## Running Tests + +All the following commands assume you have a terminal opened to the `zmk/app` directory. + +First, make sure all Python dependencies are installed: + +```sh +python3 -m pip install -r scripts/requirements.txt +``` + +Tests can then be run from Zephyr's [west](https://docs.zephyrproject.org/3.2.0/develop/west/index.html) tool with the `test` subcommand: + +```sh +west test +``` + +Running it with no arguments will run the entire test suite. You can run a single test case or test set by providing the relative path from `zmk/app/tests` to the test directory, e.g. + +```sh +# Run all tests cases in the toggle-layer set +west test toggle-layer + +# Run the toggle-layer/normal test case +west test toggle-layer/normal +``` + +Any additional arguments are passed through to [PyTest](https://docs.pytest.org/). ## Creating a New Test Set