feat(workflow): Use metadata for build

.github/workflows/build.yml

@@ -9,75 +9,19 @@ on:
     paths:
       - ".github/workflows/build.yml"
       - "app/**"
+  schedule:
+    - cron: '22 4 * * *'
 
 jobs:
   build:
+    if: ${{ always() }}
     runs-on: ubuntu-latest
     container:
       image: docker.io/zmkfirmware/zmk-build-arm:2.5
+    needs: compile-matrix
     strategy:
       matrix:
-        board:
+        include: ${{ fromJSON(needs.compile-matrix.outputs.include-list) }}
-          - bluemicro840_v1
-          - nice_nano
-          - nice_nano_v2
-          - nrfmicro_13
-          - proton_c
-        shield:
-          - bfo9000_left
-          - bfo9000_right
-          - boardsource3x4
-          - corne_left
-          - corne_right
-          - cradio_left
-          - cradio_right
-          - crbn
-          - eek
-          - helix_left
-          - helix_right
-          - iris_left
-          - iris_right
-          - jian_left
-          - jian_right
-          - jorne_left
-          - jorne_right
-          - kyria_left
-          - kyria_right
-          - lily58_left
-          - lily58_right
-          - microdox_left
-          - microdox_right
-          - nibble
-          - qaz
-          - quefrency_left
-          - quefrency_right
-          - reviung41
-          - romac
-          - romac_plus
-          - settings_reset
-          - sofle_left
-          - sofle_right
-          - splitreus62_left
-          - splitreus62_right
-          - tg4x
-          - tidbit
-        cmake-args: [""]
-        include:
-          - board: bdn9_rev2
-          - board: dz60rgb_rev1
-          - board: nrf52840_m2
-            shield: m60
-          - board: planck_rev6
-          - board: proton_c
-            shield: clueboard_california
-          - board: nice_nano_v2
-            shield: kyria_left
-            cmake-args: -DCONFIG_ZMK_DISPLAY=y
-            skip-archive: true
-          - board: nice_nano_v2
-            shield: kyria_right
-            cmake-args: -DCONFIG_ZMK_DISPLAY=y
-            skip-archive: true
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -130,3 +74,359 @@ jobs:
             build/zephyr/zmk.hex
             build/zephyr/zmk.uf2
         continue-on-error: true
+  compile-matrix:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [core-coverage, board-changes, nightly]
+    outputs:
+      include-list: ${{ steps.compile-list.outputs.result }}
+    steps:
+      - name: Join build lists
+        uses: actions/github-script@v4
+        id: compile-list
+        with:
+          script: |
+            const coreCoverage = `${{ needs.core-coverage.outputs.core-include }}`;
+            const coreCoverageArray = coreCoverage ? JSON.parse(coreCoverage) : [];
+
+            const boardChanges = `${{ needs.board-changes.outputs.boards-include }}`;
+            const boardChangesArray = boardChanges ? JSON.parse(boardChanges) : [];
+
+            const nightly = `${{ needs.nightly.outputs.nightly-include }}`;
+            const nightlyArray = nightly ? JSON.parse(nightly) : [];
+
+            const combined = [...coreCoverageArray, ...boardChangesArray, ...nightlyArray];
+
+            return [...new Map(combined.map(el => [JSON.stringify(el), el])).values()];
+  core-coverage:
+    if: ${{ needs.get-changed-files.outputs.core-changes == 'true' }}
+    runs-on: ubuntu-latest
+    needs: get-changed-files
+    outputs:
+      core-include: ${{ steps.core-list.outputs.result }} 
+    steps:
+      - uses: actions/github-script@v4
+        id: core-list
+        with:
+          script: |
+            // break out to file
+            const coreCoverage = {
+              boards: [
+                'nice_nano_v2',
+                'nrfmicro_13',
+                'proton_c'
+              ],
+              shields: [
+                'corne_left',
+                'corne_right',
+                'romac',
+                'settings_reset',
+                'tidbit'
+              ],
+              include: [
+                {
+                  "board": "bdn9_rev2"
+                },
+                {
+                  "board": "nice60"
+                },
+                {
+                  "board": "nrf52840_m2",
+                  "shield": "m60"
+                },
+                {
+                  "board": "planck_rev6"
+                },
+                {
+                  "board": "proton_c",
+                  "shield": "clueboard_california"
+                },
+                {
+                  "board": "nice_nano_v2",
+                  "shield": "kyria_left",
+                  "cmake-args": "-DCONFIG_ZMK_DISPLAY=y",
+                },
+                {
+                  "board": "nice_nano_v2",
+                  "shield": "kyria_right",
+                  "cmake-args": "-DCONFIG_ZMK_DISPLAY=y",
+                },
+                {
+                  "board": "nice_nano",
+                  "shield": "romac_plus",
+                  "cmake-args": "-DCONFIG_ZMK_RGB_UNDERGLOW=y -DCONFIG_WS2812_STRIP=y"
+                }
+              ]
+            };
+
+            let include = [];
+            
+            coreCoverage.boards.forEach(b => {
+              coreCoverage.shields.forEach(s => {
+                include.push({
+                  board: b,
+                  shield: s
+                });
+              });
+            });
+
+            return [...include, ...coreCoverage.include];
+  board-changes:
+    if: ${{ needs.get-changed-files.outputs.board-changes == 'true' }}
+    runs-on: ubuntu-latest
+    needs: [get-grouped-hardware, get-changed-files]
+    outputs:
+      boards-include: ${{ steps.boards-list.outputs.result }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14.x'
+      - name: Install js-yaml
+        run: npm install js-yaml
+      - uses: actions/github-script@v4
+        id: boards-list
+        with:
+          script: |
+            const fs = require('fs');
+            const yaml = require('js-yaml');
+
+            const changedFiles = JSON.parse(`${{ needs.get-changed-files.outputs.changed-files }}`);
+            const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`);
+            const boardChanges = new Set(changedFiles.filter(f => f.startsWith('app/boards')).map(f => f.split('/').slice(0, 4).join('/')));
+
+            include = [];
+
+            for (const bc of boardChanges) {
+              const globber = await glob.create(bc + "/*.zmk.yml");
+              const files = await globber.glob();
+
+              const aggregated = files.flatMap((f) =>
+                yaml.loadAll(fs.readFileSync(f, "utf8"))
+              );
+
+              aggregated.forEach(hm => {
+                switch (hm.type) {
+                  case "board":
+                    if (hm.features && hm.features.includes("keys")) {
+                      if (hm.siblings) {
+                        hm.siblings.forEach(sib => {
+                          include.push({
+                            board: sib
+                          });
+                        });
+                      } else {
+                        include.push({
+                          board: hm.id
+                        });
+                      }
+                    } else if (hm.exposes) {
+                      hm.exposes.forEach(i => {
+                        metadata.interconnects[i].shields.forEach(s => {
+                          if (s.siblings) {
+                            s.siblings.forEach(sib => {
+                              include.push({
+                                board: hm.id,
+                                shield: sib
+                              });
+                            });
+                          } else {
+                            include.push({
+                              board: hm.id,
+                              shield: s.id
+                            });
+                          }
+                        });
+                      });
+                    } else {
+                      console.error("Board without keys or interconnect");
+                    }
+                    break;
+                  case "shield":
+                    if (hm.features && hm.features.includes("keys")) {
+                      hm.requires.forEach(i => {
+                        metadata.interconnects[i].boards.forEach(b => {
+                          if (hm.siblings) {
+                            hm.siblings.forEach(sib => {
+                              include.push({
+                                board: b.id,
+                                shield: sib
+                              });
+                            });
+                          } else {
+                            include.push({
+                              board: b.id,
+                              shield: hm.id
+                            });
+                          }
+                        });
+                      });
+                    }
+                    break;
+                  case "interconnect":
+                    break;
+                }
+              });
+            }
+                
+            return include;
+  nightly:
+    if: ${{ github.event_name == 'schedule' }}
+    runs-on: ubuntu-latest
+    needs: get-grouped-hardware
+    outputs:
+      nightly-include: ${{ steps.nightly-list.outputs.result }}
+    steps:
+      - name: Create nightly list
+        uses: actions/github-script@v4
+        id: nightly-list
+        with:
+          script: |
+            const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`);
+            let include = [];
+
+            metadata.onboard.forEach(b => {
+              if (b.siblings) {
+                b.siblings.forEach(sib => {
+                  include.push({
+                    board: sib
+                  });
+                });
+              } else {
+                include.push({
+                  board: b.id
+                });
+              }
+            });
+
+            Object.values(metadata.interconnects).forEach(i => {
+              i.boards.forEach(b => {
+                i.shields.forEach(s => {
+                  if (s.siblings) {
+                    s.siblings.forEach(sib => {
+                      include.push({
+                        board: b.id,
+                        shield: sib
+                      });
+                    });
+                  } else {
+                    include.push({
+                      board: b.id,
+                      shield: s.id
+                    });
+                  }
+                });
+              });
+            });
+
+            return include;
+  get-grouped-hardware:
+    runs-on: ubuntu-latest
+    outputs:
+      organized-metadata: ${{ steps.organize-metadata.outputs.result }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14.x'
+      - name: Install js-yaml
+        run: npm install js-yaml
+      - name: Aggregate Metadata
+        uses: actions/github-script@v4
+        id: aggregate-metadata
+        with:
+          script: |
+            const fs = require('fs');
+            const yaml = require('js-yaml');
+
+            const globber = await glob.create("app/boards/**/*.zmk.yml");
+            const files = await globber.glob();
+
+            const aggregated = files.flatMap((f) =>
+              yaml.loadAll(fs.readFileSync(f, "utf8"))
+            );
+
+            return JSON.stringify(aggregated).replace(/\\/g,"\\\\");
+          result-encoding: string
+
+      - name: Organize Metadata
+        uses: actions/github-script@v4
+        id: organize-metadata
+        with:
+          script: |
+            const hardware = JSON.parse(`${{ steps.aggregate-metadata.outputs.result }}`);
+
+            const grouped = hardware.reduce((agg, hm) => {
+              switch (hm.type) {
+                case "board":
+                  if (hm.features && hm.features.includes("keys")) {
+                    agg.onboard.push(hm);
+                  } else if (hm.exposes) {
+                    hm.exposes.forEach((element) => {
+                      let ic = agg.interconnects[element] || {
+                        boards: [],
+                        shields: [],
+                      };
+                      ic.boards.push(hm);
+                      agg.interconnects[element] = ic;
+                    });
+                  } else {
+                    console.error("Board without keys or interconnect");
+                  }
+                  break;
+                case "shield":
+                  if (hm.features && hm.features.includes("keys")) {
+                    hm.requires.forEach((id) => {
+                      let ic = agg.interconnects[id] || { boards: [], shields: [] };
+                      ic.shields.push(hm);
+                      agg.interconnects[id] = ic;
+                    });
+                  }
+                  break;
+                case "interconnect":
+                  let ic = agg.interconnects[hm.id] || { boards: [], shields: [] };
+                  ic.interconnect = hm;
+                  agg.interconnects[hm.id] = ic;
+                  break;
+              }
+              return agg;
+            },
+            { onboard: [], interconnects: {} });
+
+            return JSON.stringify(grouped).replace(/\\/g,"\\\\");
+          result-encoding: string
+  get-changed-files:
+    if: ${{ github.event_name != 'schedule' }}
+    runs-on: ubuntu-latest
+    outputs:
+      changed-files: ${{ steps.changed-files.outputs.all }}
+      board-changes: ${{ steps.board-changes.outputs.result }}
+      core-changes: ${{ steps.core-changes.outputs.result }}
+    steps:
+      - uses: Ana06/get-changed-files@v2.0.0
+        id: changed-files
+        with:
+          format: 'json'
+      - uses: actions/github-script@v4
+        id: board-changes
+        with:
+          script: |
+            const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all }}`);
+            const boardChanges = changedFiles.filter(f => f.startsWith('app/boards'));
+            return boardChanges.length ? 'true' : 'false';
+          result-encoding: string
+      - uses: actions/github-script@v4
+        id: core-changes
+        with:
+          script: |
+            const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all }}`);
+            const boardChanges = changedFiles.filter(f => f.startsWith('app/boards'));
+            const appChanges = changedFiles.filter(f => f.startsWith('app'));
+            const ymlChanges = changedFiles.includes('.github/workflows/build.yml');
+            return boardChanges.length < appChanges.length || ymlChanges ? 'true' : 'false';
+          result-encoding: string
+