From 10f755d121bccb285499ea5e818fa1fe19da03dd Mon Sep 17 00:00:00 2001 From: Nick Winans Date: Mon, 2 Aug 2021 23:07:52 -0500 Subject: [PATCH] feat(workflow): Use metadata for build --- .github/workflows/build.yml | 422 ++++++++++++++++++++++++++++++------ 1 file changed, 361 insertions(+), 61 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b790b665..b104ae69 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,75 +9,19 @@ on: paths: - ".github/workflows/build.yml" - "app/**" + schedule: + - cron: '22 4 * * *' jobs: build: + if: ${{ always() }} runs-on: ubuntu-latest container: image: docker.io/zmkfirmware/zmk-build-arm:2.5 + needs: compile-matrix strategy: matrix: - board: - - bluemicro840_v1 - - nice_nano - - nice_nano_v2 - - nrfmicro_13 - - proton_c - shield: - - bfo9000_left - - bfo9000_right - - boardsource3x4 - - corne_left - - corne_right - - cradio_left - - cradio_right - - crbn - - eek - - helix_left - - helix_right - - iris_left - - iris_right - - jian_left - - jian_right - - jorne_left - - jorne_right - - kyria_left - - kyria_right - - lily58_left - - lily58_right - - microdox_left - - microdox_right - - nibble - - qaz - - quefrency_left - - quefrency_right - - reviung41 - - romac - - romac_plus - - settings_reset - - sofle_left - - sofle_right - - splitreus62_left - - splitreus62_right - - tg4x - - tidbit - cmake-args: [""] - include: - - board: bdn9_rev2 - - board: dz60rgb_rev1 - - board: nrf52840_m2 - shield: m60 - - board: planck_rev6 - - board: proton_c - shield: clueboard_california - - board: nice_nano_v2 - shield: kyria_left - cmake-args: -DCONFIG_ZMK_DISPLAY=y - skip-archive: true - - board: nice_nano_v2 - shield: kyria_right - cmake-args: -DCONFIG_ZMK_DISPLAY=y - skip-archive: true + include: ${{ fromJSON(needs.compile-matrix.outputs.include-list) }} steps: - name: Checkout uses: actions/checkout@v2 @@ -130,3 +74,359 @@ jobs: build/zephyr/zmk.hex build/zephyr/zmk.uf2 continue-on-error: true + compile-matrix: + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [core-coverage, board-changes, nightly] + outputs: + include-list: ${{ steps.compile-list.outputs.result }} + steps: + - name: Join build lists + uses: actions/github-script@v4 + id: compile-list + with: + script: | + const coreCoverage = `${{ needs.core-coverage.outputs.core-include }}`; + const coreCoverageArray = coreCoverage ? JSON.parse(coreCoverage) : []; + + const boardChanges = `${{ needs.board-changes.outputs.boards-include }}`; + const boardChangesArray = boardChanges ? JSON.parse(boardChanges) : []; + + const nightly = `${{ needs.nightly.outputs.nightly-include }}`; + const nightlyArray = nightly ? JSON.parse(nightly) : []; + + const combined = [...coreCoverageArray, ...boardChangesArray, ...nightlyArray]; + + return [...new Map(combined.map(el => [JSON.stringify(el), el])).values()]; + core-coverage: + if: ${{ needs.get-changed-files.outputs.core-changes == 'true' }} + runs-on: ubuntu-latest + needs: get-changed-files + outputs: + core-include: ${{ steps.core-list.outputs.result }} + steps: + - uses: actions/github-script@v4 + id: core-list + with: + script: | + // break out to file + const coreCoverage = { + boards: [ + 'nice_nano_v2', + 'nrfmicro_13', + 'proton_c' + ], + shields: [ + 'corne_left', + 'corne_right', + 'romac', + 'settings_reset', + 'tidbit' + ], + include: [ + { + "board": "bdn9_rev2" + }, + { + "board": "nice60" + }, + { + "board": "nrf52840_m2", + "shield": "m60" + }, + { + "board": "planck_rev6" + }, + { + "board": "proton_c", + "shield": "clueboard_california" + }, + { + "board": "nice_nano_v2", + "shield": "kyria_left", + "cmake-args": "-DCONFIG_ZMK_DISPLAY=y", + }, + { + "board": "nice_nano_v2", + "shield": "kyria_right", + "cmake-args": "-DCONFIG_ZMK_DISPLAY=y", + }, + { + "board": "nice_nano", + "shield": "romac_plus", + "cmake-args": "-DCONFIG_ZMK_RGB_UNDERGLOW=y -DCONFIG_WS2812_STRIP=y" + } + ] + }; + + let include = []; + + coreCoverage.boards.forEach(b => { + coreCoverage.shields.forEach(s => { + include.push({ + board: b, + shield: s + }); + }); + }); + + return [...include, ...coreCoverage.include]; + board-changes: + if: ${{ needs.get-changed-files.outputs.board-changes == 'true' }} + runs-on: ubuntu-latest + needs: [get-grouped-hardware, get-changed-files] + outputs: + boards-include: ${{ steps.boards-list.outputs.result }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: '14.x' + - name: Install js-yaml + run: npm install js-yaml + - uses: actions/github-script@v4 + id: boards-list + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + const changedFiles = JSON.parse(`${{ needs.get-changed-files.outputs.changed-files }}`); + const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`); + const boardChanges = new Set(changedFiles.filter(f => f.startsWith('app/boards')).map(f => f.split('/').slice(0, 4).join('/'))); + + include = []; + + for (const bc of boardChanges) { + const globber = await glob.create(bc + "/*.zmk.yml"); + const files = await globber.glob(); + + const aggregated = files.flatMap((f) => + yaml.loadAll(fs.readFileSync(f, "utf8")) + ); + + aggregated.forEach(hm => { + switch (hm.type) { + case "board": + if (hm.features && hm.features.includes("keys")) { + if (hm.siblings) { + hm.siblings.forEach(sib => { + include.push({ + board: sib + }); + }); + } else { + include.push({ + board: hm.id + }); + } + } else if (hm.exposes) { + hm.exposes.forEach(i => { + metadata.interconnects[i].shields.forEach(s => { + if (s.siblings) { + s.siblings.forEach(sib => { + include.push({ + board: hm.id, + shield: sib + }); + }); + } else { + include.push({ + board: hm.id, + shield: s.id + }); + } + }); + }); + } else { + console.error("Board without keys or interconnect"); + } + break; + case "shield": + if (hm.features && hm.features.includes("keys")) { + hm.requires.forEach(i => { + metadata.interconnects[i].boards.forEach(b => { + if (hm.siblings) { + hm.siblings.forEach(sib => { + include.push({ + board: b.id, + shield: sib + }); + }); + } else { + include.push({ + board: b.id, + shield: hm.id + }); + } + }); + }); + } + break; + case "interconnect": + break; + } + }); + } + + return include; + nightly: + if: ${{ github.event_name == 'schedule' }} + runs-on: ubuntu-latest + needs: get-grouped-hardware + outputs: + nightly-include: ${{ steps.nightly-list.outputs.result }} + steps: + - name: Create nightly list + uses: actions/github-script@v4 + id: nightly-list + with: + script: | + const metadata = JSON.parse(`${{ needs.get-grouped-hardware.outputs.organized-metadata }}`); + let include = []; + + metadata.onboard.forEach(b => { + if (b.siblings) { + b.siblings.forEach(sib => { + include.push({ + board: sib + }); + }); + } else { + include.push({ + board: b.id + }); + } + }); + + Object.values(metadata.interconnects).forEach(i => { + i.boards.forEach(b => { + i.shields.forEach(s => { + if (s.siblings) { + s.siblings.forEach(sib => { + include.push({ + board: b.id, + shield: sib + }); + }); + } else { + include.push({ + board: b.id, + shield: s.id + }); + } + }); + }); + }); + + return include; + get-grouped-hardware: + runs-on: ubuntu-latest + outputs: + organized-metadata: ${{ steps.organize-metadata.outputs.result }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: '14.x' + - name: Install js-yaml + run: npm install js-yaml + - name: Aggregate Metadata + uses: actions/github-script@v4 + id: aggregate-metadata + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + const globber = await glob.create("app/boards/**/*.zmk.yml"); + const files = await globber.glob(); + + const aggregated = files.flatMap((f) => + yaml.loadAll(fs.readFileSync(f, "utf8")) + ); + + return JSON.stringify(aggregated).replace(/\\/g,"\\\\"); + result-encoding: string + + - name: Organize Metadata + uses: actions/github-script@v4 + id: organize-metadata + with: + script: | + const hardware = JSON.parse(`${{ steps.aggregate-metadata.outputs.result }}`); + + const grouped = hardware.reduce((agg, hm) => { + switch (hm.type) { + case "board": + if (hm.features && hm.features.includes("keys")) { + agg.onboard.push(hm); + } else if (hm.exposes) { + hm.exposes.forEach((element) => { + let ic = agg.interconnects[element] || { + boards: [], + shields: [], + }; + ic.boards.push(hm); + agg.interconnects[element] = ic; + }); + } else { + console.error("Board without keys or interconnect"); + } + break; + case "shield": + if (hm.features && hm.features.includes("keys")) { + hm.requires.forEach((id) => { + let ic = agg.interconnects[id] || { boards: [], shields: [] }; + ic.shields.push(hm); + agg.interconnects[id] = ic; + }); + } + break; + case "interconnect": + let ic = agg.interconnects[hm.id] || { boards: [], shields: [] }; + ic.interconnect = hm; + agg.interconnects[hm.id] = ic; + break; + } + return agg; + }, + { onboard: [], interconnects: {} }); + + return JSON.stringify(grouped).replace(/\\/g,"\\\\"); + result-encoding: string + get-changed-files: + if: ${{ github.event_name != 'schedule' }} + runs-on: ubuntu-latest + outputs: + changed-files: ${{ steps.changed-files.outputs.all }} + board-changes: ${{ steps.board-changes.outputs.result }} + core-changes: ${{ steps.core-changes.outputs.result }} + steps: + - uses: Ana06/get-changed-files@v2.0.0 + id: changed-files + with: + format: 'json' + - uses: actions/github-script@v4 + id: board-changes + with: + script: | + const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all }}`); + const boardChanges = changedFiles.filter(f => f.startsWith('app/boards')); + return boardChanges.length ? 'true' : 'false'; + result-encoding: string + - uses: actions/github-script@v4 + id: core-changes + with: + script: | + const changedFiles = JSON.parse(`${{ steps.changed-files.outputs.all }}`); + const boardChanges = changedFiles.filter(f => f.startsWith('app/boards')); + const appChanges = changedFiles.filter(f => f.startsWith('app')); + const ymlChanges = changedFiles.includes('.github/workflows/build.yml'); + return boardChanges.length < appChanges.length || ymlChanges ? 'true' : 'false'; + result-encoding: string +