Merge e1a925ff83 into 6cb42a8060

.github/workflows/build-user-config.yml

@@ -29,18 +29,16 @@ jobs:
     runs-on: ubuntu-latest
     name: Fetch Build Keyboards
     outputs:
-      build_matrix: ${{ env.build_matrix }}
+      build_matrix: ${{ steps.fetch_build_matrix.outputs.result }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Install yaml2json
-        run: python3 -m pip install remarshal
-
       - name: Fetch Build Matrix
-        run: |
+        id: fetch_build_matrix
-          echo "build_matrix=$(yaml2json ${{ inputs.build_matrix_path }} | jq -c .)" >> $GITHUB_ENV
+        uses: mikefarah/yq@v4.30.8
-          yaml2json ${{ inputs.build_matrix_path }} | jq
+        with:
+          cmd: yq -o json ${{ inputs.build_matrix_path }}
 
   build:
     runs-on: ubuntu-latest

.github/workflows/test.yml

@@ -33,10 +33,11 @@ jobs:
         test: ${{ fromJSON(needs.collect-tests.outputs.test-dirs) }}
     runs-on: ubuntu-latest
     container:
-      image: docker.io/zmkfirmware/zmk-build-arm:3.0
+      image: docker.io/zmkfirmware/zmk-build-arm:stable
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - name: Cache west modules
         uses: actions/cache@v3.0.2
         env:
@@ -62,7 +63,7 @@ jobs:
         run: west zephyr-export
       - name: Test ${{ matrix.test }}
         working-directory: app
-        run: west test tests/${{ matrix.test }}
+        run: west test ${{ matrix.test }}
       - name: Archive artifacts
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

.vscode/settings.json

@@ -3,5 +3,9 @@
     "*.overlay": "dts",
     "*.keymap": "dts"
   },
-  "python.formatting.provider": "black"
+  "python.formatting.provider": "black",
+  "python.testing.pytestEnabled": true,
+  "python.testing.unittestEnabled": false,
+  "python.testing.cwd": "${workspaceFolder}/app",
+  "python.testing.pytestArgs": ["test_zmk.py", "-v", "--numprocesses=auto"]
 }

app/run-test.sh

@@ -1,47 +0,0 @@
-#!/bin/sh
-
-# Copyright (c) 2020 The ZMK Contributors
-# SPDX-License-Identifier: MIT
-
-if [ -z "$1" ]; then
-	echo "Usage: ./run-test.sh <path to testcase>"
-	exit 1
-fi
-
-path="$1"
-if [ $path = "all" ]; then
-	path="tests"
-fi
-
-testcases=$(find $path -name native_posix_64.keymap -exec dirname \{\} \;)
-num_cases=$(echo "$testcases" | wc -l)
-if [ $num_cases -gt 1 ] || [ "$testcases" != "$path" ]; then
-	echo "" > ./build/tests/pass-fail.log
-	echo "$testcases" | xargs -L 1 -P ${J:-4} ./run-test.sh
-	err=$?
-	sort -k2 ./build/tests/pass-fail.log
-	exit $err
-fi
-
-testcase="$path"
-echo "Running $testcase:"
-
-west build -d build/$testcase -b native_posix_64 -- -DZMK_CONFIG="$(pwd)/$testcase" > /dev/null 2>&1
-if [ $? -gt 0 ]; then
-	echo "FAILED: $testcase did not build" | tee -a ./build/tests/pass-fail.log
-	exit 1
-fi
-
-./build/$testcase/zephyr/zmk.exe | sed -e "s/.*> //" | tee build/$testcase/keycode_events_full.log | sed -n -f $testcase/events.patterns > build/$testcase/keycode_events.log
-diff -au $testcase/keycode_events.snapshot build/$testcase/keycode_events.log
-if [ $? -gt 0 ]; then
-	if [ -f $testcase/pending ]; then
-		echo "PENDING: $testcase" | tee -a ./build/tests/pass-fail.log
-		exit 0
-	fi
-	echo "FAILED: $testcase" | tee -a ./build/tests/pass-fail.log
-	exit 1
-fi
-
-echo "PASS: $testcase" | tee -a ./build/tests/pass-fail.log
-exit 0

app/scripts/requirements.txt

@@ -1,8 +1,10 @@
 # Copyright (c) 2021 The ZMK Contributors
 # SPDX-License-Identifier: MIT
 
-# Convert YAML to JSON for validation
-remarshal>=0.14.0
-
 # Perform our hardware metadata validation
-jsonschema>=3.2.0
+jsonschema>=3.2.0
+
+# Unit testing
+pythonsed>=2.1
+pytest>=7.0.0
+pytest-xdist>=3.0.0

app/scripts/west_commands/test.py

@@ -2,9 +2,7 @@
 # SPDX-License-Identifier: MIT
 """Test runner for ZMK."""
 
-import os
+import pytest
-import subprocess
-
 from west.commands import WestCommand
 from west import log  # use this for user output
 
@@ -14,7 +12,8 @@ class Test(WestCommand):
         super().__init__(
             name="test",
             help="run ZMK testsuite",
-            description="Run the ZMK testsuite.",
+            description="Run the ZMK testsuite. Arguments are passed through to pytest.",
+            accepts_unknown_args=True,
         )
 
     def do_add_parser(self, parser_adder):
@@ -25,17 +24,18 @@ class Test(WestCommand):
         )
 
         parser.add_argument(
-            "test_path",
+            "test",
-            default="all",
+            help="The path to the test to run. Runs all tests if not specified.",
-            help='The path to the test. Defaults to "all".',
             nargs="?",
         )
         return parser
 
     def do_run(self, args, unknown_args):
-        # the run-test script assumes the app directory is the current dir.
+        pytest_args = [f"{self.topdir}/app/test_zmk.py", "--numprocesses=auto"]
-        os.chdir(f"{self.topdir}/app")
+        pytest_args += unknown_args
-        completed_process = subprocess.run(
+
-            [f"{self.topdir}/app/run-test.sh", args.test_path]
+        if args.test:
-        )
+            pytest_args += ["-k", args.test]
-        exit(completed_process.returncode)
+
+        returncode = pytest.main(pytest_args)
+        exit(returncode)

app/test_zmk.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 The ZMK Contributors
+# SPDX-License-Identifier: MIT
+
+
+from io import StringIO
+from pathlib import Path
+import subprocess
+from subprocess import CalledProcessError, PIPE
+from typing import Sequence, Union
+
+from PythonSed import Sed
+import pytest
+
+
+APP_PATH = Path(__file__).parent
+TESTS_PATH = APP_PATH / "tests"
+BUILD_PATH = APP_PATH / "build/tests"
+
+# TODO: change all tests to build with a unit test shield
+TEST_KEYMAP_NAME = "native_posix_64.keymap"
+
+
+def check_call_teed(args: Sequence[Union[str, Path]], **kwargs):
+    """
+    Call a subprocess, print its output, and also return its output.
+    """
+    with subprocess.Popen(
+        args, bufsize=1, stdout=PIPE, text=True, encoding="utf-8", **kwargs
+    ) as p:
+        stdout = StringIO()
+
+        for line in p.stdout:
+            stdout.write(line)
+            print(line, end="")
+
+        returncode = p.wait()
+        output = stdout.getvalue()
+
+        if returncode:
+            raise CalledProcessError(returncode, p.args, output)
+
+        return output
+
+
+def strip_log_tags(log: str):
+    def strip(line: str):
+        left, _, right = line.partition("> ")
+        return right or left
+
+    return "\n".join(strip(line) for line in log.splitlines())
+
+
+def filter_log(log: str, patterns_file: Path):
+    sed = Sed(encoding="utf-8")
+    sed.no_autoprint = True
+    sed.load_script(str(patterns_file))
+
+    input = StringIO(log)
+    output = StringIO()
+    sed.apply(input, output)
+
+    return output.getvalue()
+
+
+class Runner:
+    def __init__(self, path: Path, build_path: Path):
+        self.path = path
+        self.build_path = build_path
+
+    @property
+    def board(self) -> str:
+        """The Zephyr board to build"""
+        raise NotImplementedError()
+
+    @classmethod
+    def check(cls):
+        """Check that the environment is valid for the runner."""
+        pass
+
+    def run(self) -> str:
+        """Run the unit test and return its output."""
+        raise NotImplementedError()
+
+
+class PosixRunner(Runner):
+    @property
+    def board(self):
+        return "native_posix_64"
+
+    def run(self):
+        return check_call_teed([self.build_path / "zephyr/zmk.exe"], cwd=APP_PATH)
+
+
+# TODO: Add QemuRunner for non-Posix platforms
+
+
+class ZmkTestCase:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.runner = self._get_runner()
+        self.runner.check()
+
+        self.patterns_file = self.path / "events.patterns"
+        self.snapshot_file = self.path / "keycode_events.snapshot"
+
+        if not self.patterns_file.exists():
+            pytest.fail(f"Missing patterns file: {self.patterns_file}")
+
+        if not self.snapshot_file.exists():
+            pytest.fail(f"Missing snapshot file: {self.snapshot_file}")
+
+    def run(self):
+        if reason := self.get_pending_reason():
+            pytest.skip(reason)
+
+        self._build()
+        self._test()
+
+    @property
+    def build_path(self):
+        return BUILD_PATH / self.path.relative_to(TESTS_PATH)
+
+    @property
+    def relative_build_path(self):
+        return self.build_path.relative_to(APP_PATH)
+
+    def get_pending_reason(self):
+        for file in ["pending", f"pending-{self.runner.board}"]:
+            try:
+                return (self.path / file).read_text(encoding="utf-8")
+            except:
+                pass
+
+        return None
+
+    def _build(self):
+        subprocess.check_call(
+            [
+                "west",
+                "build",
+                "-d",
+                self.build_path.relative_to(APP_PATH).as_posix(),
+                "-b",
+                self.runner.board,
+                "--",
+                f"-DZMK_CONFIG={self.path.as_posix()}",
+            ],
+            cwd=APP_PATH,
+        )
+
+    def _test(self):
+        output = self.runner.run()
+        output = strip_log_tags(output)
+        with self._open_log("keycode_events_full.log") as log:
+            log.write(output)
+
+        output = filter_log(output, self.patterns_file)
+        with self._open_log("keycode_events.log") as log:
+            log.write(output)
+
+        assert output == self.snapshot_file.read_text(encoding="utf-8")
+
+    def _get_runner(self) -> type[Runner]:
+        # TODO: return a QemuRunner for non-Posix platforms
+        return PosixRunner(self.path, self.build_path)
+
+    def _open_log(self, name: str, mode="w"):
+        log_path = self.build_path / name
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+
+        return log_path.open(mode, encoding="utf-8")
+
+
+def get_tests():
+    paths = sorted(keymap.parent for keymap in TESTS_PATH.rglob(TEST_KEYMAP_NAME))
+
+    return [
+        pytest.param(path, id=str(path.relative_to(TESTS_PATH).as_posix()))
+        for path in paths
+    ]
+
+
+@pytest.mark.parametrize("name", get_tests())
+def test(name):
+    ZmkTestCase(TESTS_PATH / name).run()

docs/docs/development/tests.md

@@ -5,8 +5,34 @@ sidebar_label: Tests
 
 - Running tests requires [native posix support](posix-board.md).
 - Any folder under `/app/tests` containing `native_posix_64.keymap` will be selected when running `west test`.
-- Run tests from within the `/zmk/app` directory.
+
-- Run a single test with `west test <testname>`, like `west test tests/toggle-layer/normal`.
+## Running Tests
+
+All the following commands assume you have a terminal opened to the `zmk/app` directory.
+
+First, make sure all Python dependencies are installed:
+
+```sh
+python3 -m pip install -r scripts/requirements.txt
+```
+
+Tests can then be run from Zephyr's [west](https://docs.zephyrproject.org/3.2.0/develop/west/index.html) tool with the `test` subcommand:
+
+```sh
+west test
+```
+
+Running it with no arguments will run the entire test suite. You can run a single test case or test set by providing the relative path from `zmk/app/tests` to the test directory, e.g.
+
+```sh
+# Run all tests cases in the toggle-layer set
+west test toggle-layer
+
+# Run the toggle-layer/normal test case
+west test toggle-layer/normal
+```
+
+Any additional arguments are passed through to [PyTest](https://docs.pytest.org/).
 
 ## Creating a New Test Set