refactor(test): Run tests with pytest

Replaced the run-test.sh script with a Python script that runs our unit
tests through pytest. Tests are now run in parallel to speed up running
the entire test suite, and it allows for integration with other tools
that support pytest, such as IDEs.

Also removed a dependency on remarshal, because it depends on an old
version of PyYAML that conflicts with other Python packages in our
Docker image. Replaced it with yq.

.github/workflows/build-user-config.yml

@@ -29,18 +29,16 @@ jobs:
     runs-on: ubuntu-latest
     name: Fetch Build Keyboards
     outputs:
-      build_matrix: ${{ env.build_matrix }}
+      build_matrix: ${{ steps.fetch_build_matrix.outputs.result }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install yaml2json
-        run: python3 -m pip install remarshal
-
       - name: Fetch Build Matrix
-        run: |
-          echo "build_matrix=$(yaml2json ${{ inputs.build_matrix_path }} | jq -c .)" >> $GITHUB_ENV
-          yaml2json ${{ inputs.build_matrix_path }} | jq
+        id: fetch_build_matrix
+        uses: mikefarah/yq@v4.30.8
+        with:
+          cmd: yq -o json ${{ inputs.build_matrix_path }}
 
   build:
     runs-on: ubuntu-latest

.github/workflows/test.yml

@@ -33,10 +33,11 @@ jobs:
         test: ${{ fromJSON(needs.collect-tests.outputs.test-dirs) }}
     runs-on: ubuntu-latest
     container:
-      image: docker.io/zmkfirmware/zmk-build-arm:3.0
+      image: docker.io/zmkfirmware/zmk-build-arm:stable
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - name: Cache west modules
         uses: actions/cache@v3.0.2
         env:
@@ -62,7 +63,7 @@ jobs:
         run: west zephyr-export
       - name: Test ${{ matrix.test }}
         working-directory: app
-        run: west test tests/${{ matrix.test }}
+        run: west test ${{ matrix.test }}
       - name: Archive artifacts
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

.vscode/settings.json

@@ -3,5 +3,9 @@
     "*.overlay": "dts",
     "*.keymap": "dts"
   },
-  "python.formatting.provider": "black"
+  "python.formatting.provider": "black",
+  "python.testing.pytestEnabled": true,
+  "python.testing.unittestEnabled": false,
+  "python.testing.cwd": "${workspaceFolder}/app",
+  "python.testing.pytestArgs": ["test_zmk.py", "-v", "--numprocesses=auto"]
 }

app/run-test.sh

@@ -1,47 +0,0 @@
-#!/bin/sh
-
-# Copyright (c) 2020 The ZMK Contributors
-# SPDX-License-Identifier: MIT
-
-if [ -z "$1" ]; then
-	echo "Usage: ./run-test.sh <path to testcase>"
-	exit 1
-fi
-
-path="$1"
-if [ $path = "all" ]; then
-	path="tests"
-fi
-
-testcases=$(find $path -name native_posix_64.keymap -exec dirname \{\} \;)
-num_cases=$(echo "$testcases" | wc -l)
-if [ $num_cases -gt 1 ] || [ "$testcases" != "$path" ]; then
-	echo "" > ./build/tests/pass-fail.log
-	echo "$testcases" | xargs -L 1 -P ${J:-4} ./run-test.sh
-	err=$?
-	sort -k2 ./build/tests/pass-fail.log
-	exit $err
-fi
-
-testcase="$path"
-echo "Running $testcase:"
-
-west build -d build/$testcase -b native_posix_64 -- -DZMK_CONFIG="$(pwd)/$testcase" > /dev/null 2>&1
-if [ $? -gt 0 ]; then
-	echo "FAILED: $testcase did not build" | tee -a ./build/tests/pass-fail.log
-	exit 1
-fi
-
-./build/$testcase/zephyr/zmk.exe | sed -e "s/.*> //" | tee build/$testcase/keycode_events_full.log | sed -n -f $testcase/events.patterns > build/$testcase/keycode_events.log
-diff -au $testcase/keycode_events.snapshot build/$testcase/keycode_events.log
-if [ $? -gt 0 ]; then
-	if [ -f $testcase/pending ]; then
-		echo "PENDING: $testcase" | tee -a ./build/tests/pass-fail.log
-		exit 0
-	fi
-	echo "FAILED: $testcase" | tee -a ./build/tests/pass-fail.log
-	exit 1
-fi
-
-echo "PASS: $testcase" | tee -a ./build/tests/pass-fail.log
-exit 0

app/scripts/requirements.txt

@@ -1,8 +1,10 @@
 # Copyright (c) 2021 The ZMK Contributors
 # SPDX-License-Identifier: MIT
 
-# Convert YAML to JSON for validation
-remarshal>=0.14.0
-
 # Perform our hardware metadata validation
-jsonschema>=3.2.0
+jsonschema>=3.2.0
+
+# Unit testing
+pythonsed>=2.1
+pytest>=7.0.0
+pytest-xdist>=3.0.0

app/scripts/west_commands/test.py

@@ -2,9 +2,7 @@
 # SPDX-License-Identifier: MIT
 """Test runner for ZMK."""
 
-import os
-import subprocess
-
+import pytest
 from west.commands import WestCommand
 from west import log  # use this for user output
 
@@ -14,7 +12,8 @@ class Test(WestCommand):
         super().__init__(
             name="test",
             help="run ZMK testsuite",
-            description="Run the ZMK testsuite.",
+            description="Run the ZMK testsuite. Arguments are passed through to pytest.",
+            accepts_unknown_args=True,
         )
 
     def do_add_parser(self, parser_adder):
@@ -25,17 +24,18 @@ class Test(WestCommand):
         )
 
         parser.add_argument(
-            "test_path",
-            default="all",
-            help='The path to the test. Defaults to "all".',
+            "test",
+            help="The path to the test to run. Runs all tests if not specified.",
             nargs="?",
         )
         return parser
 
     def do_run(self, args, unknown_args):
-        # the run-test script assumes the app directory is the current dir.
-        os.chdir(f"{self.topdir}/app")
-        completed_process = subprocess.run(
-            [f"{self.topdir}/app/run-test.sh", args.test_path]
-        )
-        exit(completed_process.returncode)
+        pytest_args = [f"{self.topdir}/app/test_zmk.py", "--numprocesses=auto"]
+        pytest_args += unknown_args
+
+        if args.test:
+            pytest_args += ["-k", args.test]
+
+        returncode = pytest.main(pytest_args)
+        exit(returncode)

app/test_zmk.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 The ZMK Contributors
+# SPDX-License-Identifier: MIT
+
+
+from io import StringIO
+from pathlib import Path
+import subprocess
+from subprocess import CalledProcessError, PIPE
+from typing import Sequence, Union
+
+from PythonSed import Sed
+import pytest
+
+
+APP_PATH = Path(__file__).parent
+TESTS_PATH = APP_PATH / "tests"
+BUILD_PATH = APP_PATH / "build/tests"
+
+# TODO: change all tests to build with a unit test shield
+TEST_KEYMAP_NAME = "native_posix_64.keymap"
+
+
+def check_call_teed(args: Sequence[Union[str, Path]], **kwargs):
+    """
+    Call a subprocess, print its output, and also return its output.
+    """
+    with subprocess.Popen(
+        args, bufsize=1, stdout=PIPE, text=True, encoding="utf-8", **kwargs
+    ) as p:
+        stdout = StringIO()
+
+        for line in p.stdout:
+            stdout.write(line)
+            print(line, end="")
+
+        returncode = p.wait()
+        output = stdout.getvalue()
+
+        if returncode:
+            raise CalledProcessError(returncode, p.args, output)
+
+        return output
+
+
+def strip_log_tags(log: str):
+    def strip(line: str):
+        left, _, right = line.partition("> ")
+        return right or left
+
+    return "\n".join(strip(line) for line in log.splitlines())
+
+
+def filter_log(log: str, patterns_file: Path):
+    sed = Sed(encoding="utf-8")
+    sed.no_autoprint = True
+    sed.load_script(str(patterns_file))
+
+    input = StringIO(log)
+    output = StringIO()
+    sed.apply(input, output)
+
+    return output.getvalue()
+
+
+class Runner:
+    def __init__(self, path: Path, build_path: Path):
+        self.path = path
+        self.build_path = build_path
+
+    @property
+    def board(self) -> str:
+        """The Zephyr board to build"""
+        raise NotImplementedError()
+
+    @classmethod
+    def check(cls):
+        """Check that the environment is valid for the runner."""
+        pass
+
+    def run(self) -> str:
+        """Run the unit test and return its output."""
+        raise NotImplementedError()
+
+
+class PosixRunner(Runner):
+    @property
+    def board(self):
+        return "native_posix_64"
+
+    def run(self):
+        return check_call_teed([self.build_path / "zephyr/zmk.exe"], cwd=APP_PATH)
+
+
+# TODO: Add QemuRunner for non-Posix platforms
+
+
+class ZmkTestCase:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.runner = self._get_runner()
+        self.runner.check()
+
+        self.patterns_file = self.path / "events.patterns"
+        self.snapshot_file = self.path / "keycode_events.snapshot"
+
+        if not self.patterns_file.exists():
+            pytest.fail(f"Missing patterns file: {self.patterns_file}")
+
+        if not self.snapshot_file.exists():
+            pytest.fail(f"Missing snapshot file: {self.snapshot_file}")
+
+    def run(self):
+        if reason := self.get_pending_reason():
+            pytest.skip(reason)
+
+        self._build()
+        self._test()
+
+    @property
+    def build_path(self):
+        return BUILD_PATH / self.path.relative_to(TESTS_PATH)
+
+    @property
+    def relative_build_path(self):
+        return self.build_path.relative_to(APP_PATH)
+
+    def get_pending_reason(self):
+        for file in ["pending", f"pending-{self.runner.board}"]:
+            try:
+                return (self.path / file).read_text(encoding="utf-8")
+            except:
+                pass
+
+        return None
+
+    def _build(self):
+        subprocess.check_call(
+            [
+                "west",
+                "build",
+                "-d",
+                self.build_path.relative_to(APP_PATH).as_posix(),
+                "-b",
+                self.runner.board,
+                "--",
+                f"-DZMK_CONFIG={self.path.as_posix()}",
+            ],
+            cwd=APP_PATH,
+        )
+
+    def _test(self):
+        output = self.runner.run()
+        output = strip_log_tags(output)
+        with self._open_log("keycode_events_full.log") as log:
+            log.write(output)
+
+        output = filter_log(output, self.patterns_file)
+        with self._open_log("keycode_events.log") as log:
+            log.write(output)
+
+        assert output == self.snapshot_file.read_text(encoding="utf-8")
+
+    def _get_runner(self) -> type[Runner]:
+        # TODO: return a QemuRunner for non-Posix platforms
+        return PosixRunner(self.path, self.build_path)
+
+    def _open_log(self, name: str, mode="w"):
+        log_path = self.build_path / name
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+
+        return log_path.open(mode, encoding="utf-8")
+
+
+def get_tests():
+    paths = sorted(keymap.parent for keymap in TESTS_PATH.rglob(TEST_KEYMAP_NAME))
+
+    return [
+        pytest.param(path, id=str(path.relative_to(TESTS_PATH).as_posix()))
+        for path in paths
+    ]
+
+
+@pytest.mark.parametrize("name", get_tests())
+def test(name):
+    ZmkTestCase(TESTS_PATH / name).run()

docs/docs/development/tests.md

@@ -5,8 +5,34 @@ sidebar_label: Tests
 
 - Running tests requires [native posix support](posix-board.md).
 - Any folder under `/app/tests` containing `native_posix_64.keymap` will be selected when running `west test`.
-- Run tests from within the `/zmk/app` directory.
-- Run a single test with `west test <testname>`, like `west test tests/toggle-layer/normal`.
+
+## Running Tests
+
+All the following commands assume you have a terminal opened to the `zmk/app` directory.
+
+First, make sure all Python dependencies are installed:
+
+```sh
+python3 -m pip install -r scripts/requirements.txt
+```
+
+Tests can then be run from Zephyr's [west](https://docs.zephyrproject.org/3.2.0/develop/west/index.html) tool with the `test` subcommand:
+
+```sh
+west test
+```
+
+Running it with no arguments will run the entire test suite. You can run a single test case or test set by providing the relative path from `zmk/app/tests` to the test directory, e.g.
+
+```sh
+# Run all tests cases in the toggle-layer set
+west test toggle-layer
+
+# Run the toggle-layer/normal test case
+west test toggle-layer/normal
+```
+
+Any additional arguments are passed through to [PyTest](https://docs.pytest.org/).
 
 ## Creating a New Test Set