Docker container and lambda function for performing firmware builds

Provides an entry point that builds and returns a combined LH + RH keyboard
firmware when provided a keymap via a POST body.

Wraps compilation with ccache, and includes a pre-warmed cache of the build in /tmp/ccache.
To maximize chance of a direct cache hit, changes the lambda driver to always build in /tmp/build.

some back of the envelope measurements (2012 xeon e3-1230v2, nixos)
clean build, no cache -> 21.308
clean build, cache -> 7.145
modified keymap, clean build, cache -> 12.127

.github/workflows/build-container.yml

@@ -0,0 +1,72 @@
+name: Build Compiler Service Container
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'moergo-sc/zmk'
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      ECR_REPOSITORY: zmk-builder-lambda
+      UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV
+      REVISION_TAG: ${{ github.sha }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder
+          aws-region: us-east-1
+      - name: Extract image tag name
+        shell: bash
+        run: |
+          if [ "$GITHUB_REF" = "refs/heads/main" ]; then
+            tag="latest"
+          elif [ "$GITHUB_HEAD_REF" ]; then
+            pr=${GITHUB_REF#refs/pull/}
+            pr=${pr%/merge}
+            tag="pr${pr}.${GITHUB_HEAD_REF}"
+          elif [[ "$GITHUB_REF" == refs/tags/* ]]; then
+            tag="${GITHUB_REF#refs/tags/}"
+          else
+            echo "Not a release branch or tag" >&2
+            exit 1
+          fi
+          # Replace / with . in container tag names
+          tag="${tag//\//.}"
+          echo "IMAGE_TAG=${tag}" >> $GITHUB_ENV
+        id: extract_name
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+      - uses: cachix/install-nix-action@v16
+        with:
+          nix_path: nixpkgs=channel:nixos-22.05
+      - uses: cachix/cachix-action@v12
+        with:
+          name: moergo-glove80-zmk-dev
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+      - name: Build lambda image
+        run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A directLambdaImage -o directLambdaImage
+      - name: Import OCI image into docker-daemon
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: skopeo --insecure-policy copy oci:directLambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+      - name: Push container image to Amazon ECR
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: docker push $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+      - name: Notify the build pipeline that the compile containers have updated
+        run: aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION /dev/null

.github/workflows/cleanup-container.yml

@@ -0,0 +1,38 @@
+name: Clean up PR Compiler Service Container
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'moergo-sc/zmk'
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      ECR_REPOSITORY: zmk-builder-lambda
+      UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV
+      PR_NUMBER: ${{ github.event.number }}
+    steps:
+      - name: Extract image tag name
+        shell: bash
+        run: |
+          tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}"
+          # Replace / with . in container tag names
+          tag="${tag//\//.}"
+          echo "IMAGE_TAG=${tag}" >> $GITHUB_ENV
+        id: extract_name
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder
+          aws-region: us-east-1
+      - name: Delete the container image for the PR from ECR
+        run: aws ecr batch-delete-image --repository-name $ECR_REPOSITORY --image-ids imageTag=$IMAGE_TAG
+      - name: Notify the build pipeline that the compile containers have updated
+        run: aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION /dev/null

.github/workflows/nix-build.yml

@@ -1,4 +1,4 @@
-name: Build
+name: Build Glove80 Firmware
 
 on:
   push:

lambda/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gem 'aws_lambda_ric'
+gem 'rack'
+gem 'sinatra', '~> 2'
+
+# The version on rubygems (1.0.7) is very out of date
+gem 'serverless-rack', git: 'https://github.com/logandk/serverless-rack', branch: '7364305bc'

lambda/Gemfile.lock

@@ -0,0 +1,36 @@
+GIT
+  remote: https://github.com/logandk/serverless-rack
+  revision: 7364305bcbbf7f6cc6851497069a5a4cb91936b1
+  branch: 7364305bc
+  specs:
+    serverless-rack (1.0.7)
+      rack (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws_lambda_ric (2.0.0)
+    mustermann (2.0.2)
+      ruby2_keywords (~> 0.0.1)
+    rack (2.2.4)
+    rack-protection (2.2.2)
+      rack
+    ruby2_keywords (0.0.5)
+    sinatra (2.2.2)
+      mustermann (~> 2.0)
+      rack (~> 2.2)
+      rack-protection (= 2.2.2)
+      tilt (~> 2.0)
+    tilt (2.0.11)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws_lambda_ric
+  rack
+  serverless-rack!
+  sinatra (~> 2)
+
+BUNDLED WITH
+   2.1.4

lambda/app.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'rack'
+require 'serverless_rack'
+
+require './web_app'
+require './compiler'
+
+$app = Rack::Builder.new do
+  run WebApp
+end.to_app
+
+module LambdaFunction
+  # Handle a API Gateway/ALB-structured HTTP request using the Sinatra app
+  class HttpHandler
+    def self.process(event:, context:)
+      handle_request(app: $app, event: event, context: context)
+    end
+  end
+
+  # Handle a non-HTTP proxied request, returning either the compiled result or
+  # an error as JSON.
+  class DirectHandler
+    REVISION = ENV.fetch('REVISION', 'unknown')
+
+    def self.process(event:, context:)
+      return { type: 'keep_alive' } if event.has_key?('keep_alive')
+
+      keymap_data = event.fetch('keymap') do
+        return error(status: 400, message: 'Missing required argument: keymap')
+      end
+
+      keymap_data =
+        begin
+          Base64.strict_decode64(keymap_data)
+        rescue ArgumentError
+          return error(status: 400, message: 'Invalid Base64 in keymap input')
+        end
+
+      result, log =
+        begin
+          Compiler.new.compile(keymap_data)
+        rescue Compiler::CompileError => e
+          return error(status: e.status, message: e.message, detail: e.log)
+        end
+
+      result = Base64.strict_encode64(result)
+
+      { type: 'result', result: result, log: log, revision: REVISION }
+    rescue StandardError => e
+      error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message])
+    end
+
+    def self.error(status:, message:, detail: nil)
+      { type: 'error', status: status, message: message, detail: detail, revision: REVISION }
+    end
+  end
+end

lambda/compiler.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+require 'json'
+require 'base64'
+
+class Compiler
+  class CompileError < RuntimeError
+    attr_reader :status, :log
+
+    def initialize(message, status: 400, log:)
+      super(message)
+      @status = status
+      @log = log
+    end
+  end
+
+  def compile(keymap_data)
+    in_build_dir do
+      File.open('build.keymap', 'w') do |io|
+        io.write(keymap_data)
+      end
+
+      compile_output = nil
+
+      IO.popen(['compileZmk', './build.keymap'], err: [:child, :out]) do |io|
+        compile_output = io.read
+      end
+
+      compile_output = compile_output.split("\n")
+
+      unless $?.success?
+        status = $?.exitstatus
+        raise CompileError.new("Compile failed with exit status #{status}", log: compile_output)
+      end
+
+      unless File.exist?('zephyr/combined.uf2')
+        raise CompileError.new('Compile failed to produce result binary', status: 500, log: compile_output)
+      end
+
+      result = File.read('zephyr/combined.uf2')
+
+      [result, compile_output]
+    end
+  end
+
+  # Lambda is single-process per container, and we get substantial speedups
+  # from ccache by always building in the same path
+  BUILD_DIR = '/tmp/build'
+
+  def in_build_dir
+    FileUtils.remove_entry(BUILD_DIR, true)
+    Dir.mkdir(BUILD_DIR)
+    Dir.chdir(BUILD_DIR)
+    yield
+  ensure
+    FileUtils.remove_entry(BUILD_DIR, true) rescue nil
+  end
+end

lambda/default.nix

@@ -0,0 +1,26 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs;
+
+let
+  bundleEnv = bundlerEnv {
+    name = "lambda-bundler-env";
+    ruby     = ruby_3_1;
+    gemfile  = ./Gemfile;
+    lockfile = ./Gemfile.lock;
+    gemset   = ./gemset.nix;
+  };
+
+  source = stdenv.mkDerivation {
+    name    = "lambda-builder";
+    version = "0.0.1";
+    src = ./.;
+    installPhase = ''
+      cp -r ./ $out
+    '';
+  };
+
+in
+{
+  inherit bundleEnv source;
+}

lambda/gemset.nix

@@ -0,0 +1,88 @@
+{
+  aws_lambda_ric = {
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "19c4xlgnhgwf3n3z57z16nmr76jd2vihhshknm5zqip2g00awhi1";
+      type = "gem";
+    };
+    version = "2.0.0";
+  };
+  mustermann = {
+    dependencies = ["ruby2_keywords"];
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0m70qz27mlv2rhk4j1li6pw797gmiwwqg02vcgxcxr1rq2v53rnb";
+      type = "gem";
+    };
+    version = "2.0.2";
+  };
+  rack = {
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0axc6w0rs4yj0pksfll1hjgw1k6a5q0xi2lckh91knfb72v348pa";
+      type = "gem";
+    };
+    version = "2.2.4";
+  };
+  rack-protection = {
+    dependencies = ["rack"];
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "169jzzgvbjrqmz4q55wp9pg4ji2h90mggcdxy152gv5vp96l2hgx";
+      type = "gem";
+    };
+    version = "2.2.2";
+  };
+  ruby2_keywords = {
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "1vz322p8n39hz3b4a9gkmz9y7a5jaz41zrm2ywf31dvkqm03glgz";
+      type = "gem";
+    };
+    version = "0.0.5";
+  };
+  serverless-rack = {
+    dependencies = ["rack"];
+    groups = ["default"];
+    platforms = [];
+    source = {
+      fetchSubmodules = false;
+      rev = "7364305bcbbf7f6cc6851497069a5a4cb91936b1";
+      sha256 = "0c7ch0s0nl70p6ijg7q0jnq8ca2rhp5wqfp91kai81dy7d71mq65";
+      type = "git";
+      url = "https://github.com/logandk/serverless-rack";
+    };
+    version = "1.0.7";
+  };
+  sinatra = {
+    dependencies = ["mustermann" "rack" "rack-protection" "tilt"];
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "0mbjp75dy35q796iard8izsy7gk55g2c3q864r2p13my3yjmlcvz";
+      type = "gem";
+    };
+    version = "2.2.2";
+  };
+  tilt = {
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "186nfbcsk0l4l86gvng1fw6jq6p6s7rc0caxr23b3pnbfb20y63v";
+      type = "gem";
+    };
+    version = "2.0.11";
+  };
+}

lambda/shell.nix

@@ -0,0 +1,9 @@
+{ pkgs ? (import <nixpkgs> {})}:
+
+let
+  lambda = import ./default.nix { inherit pkgs; };
+in
+pkgs.stdenv.mkDerivation {
+  name = "lambda-shell";
+  buildInputs = [lambda.bundleEnv.wrappedRuby];
+}

lambda/web_app.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'sinatra/base'
+require './compiler'
+
+class WebApp < Sinatra::Base
+  set :environment, :production
+  set :show_exceptions, false
+  set :logging, nil
+  set :default_content_type, 'application/json'
+
+  def json_body(hash)
+    body(hash.to_json)
+  end
+
+  post '/api/compile' do
+    request.body.rewind
+    keymap_data = request.body.read
+    result, log = Compiler.new.compile(keymap_data)
+
+    status 200
+    content_type 'application/octet-stream'
+    headers 'X-Debug-Output': log.to_json
+    body result
+  end
+
+  error Compiler::CompileError do
+    e = env['sinatra.error']
+    status e.status
+    json_body(error: e.message, detail: e.log)
+  end
+
+  error do
+    e = env['sinatra.error']
+    status 500
+    json_body(error: "Unexpected error: #{e.class}", detail: [e.message])
+  end
+
+  not_found do
+    status 404
+    json_body(error: 'No such path', detail: nil)
+  end
+end

nix/ccache.nix

@@ -0,0 +1,43 @@
+{ stdenv, lib, makeWrapper, ccache
+, unwrappedCC ? stdenv.cc.cc, extraConfig ? "" }:
+
+# copied from ccache in nixpkgs, modified to glob over prefixes. Also doesn't
+# pass lib. Why was it passing lib?
+stdenv.mkDerivation {
+  name = "ccache-links";
+  passthru = {
+    isClang = unwrappedCC.isClang or false;
+    isGNU = unwrappedCC.isGNU or false;
+  };
+  nativeBuildInputs = [ makeWrapper ];
+  buildCommand = ''
+    mkdir -p $out/bin
+
+    wrap() {
+      local cname="$(basename $1)"
+      if [ -x "${unwrappedCC}/bin/$cname" ]; then
+        echo "Wrapping $1"
+        makeWrapper ${ccache}/bin/ccache $out/bin/$cname \
+          --run ${lib.escapeShellArg extraConfig} \
+          --add-flags ${unwrappedCC}/bin/$cname
+      fi
+    }
+
+    wrapAll() {
+      for prog in "$@"; do
+        wrap "$prog"
+      done
+    }
+
+    wrapAll ${unwrappedCC}/bin/{*cc,*c++,*gcc,*g++,*clang,*clang++}
+
+    for executable in $(ls ${unwrappedCC}/bin); do
+      if [ ! -x "$out/bin/$executable" ]; then
+        ln -s ${unwrappedCC}/bin/$executable $out/bin/$executable
+      fi
+    done
+    for file in $(ls ${unwrappedCC} | grep -vw bin); do
+      ln -s ${unwrappedCC}/$file $out/$file
+    done
+  '';
+}

nix/zmk.nix

@@ -66,7 +66,9 @@ stdenvNoCC.mkDerivation {
         # Transient state
         relPath == "build" || relPath == ".west" ||
         # Fetched by west
-        relPath == "modules" || relPath == "tools" || relPath == "zephyr"
+        relPath == "modules" || relPath == "tools" || relPath == "zephyr" ||
+        # Not part of ZMK
+        relPath == "lambda" || relPath == ".github"
       );
     };
 

release.nix

@@ -0,0 +1,153 @@
+{ pkgs ? (import ./nix/pinned-nixpkgs.nix {}), revision ? "HEAD" }:
+
+let
+  lib = pkgs.lib;
+  zmkPkgs = (import ./default.nix { inherit pkgs; });
+  lambda  = (import ./lambda { inherit pkgs; });
+  ccacheWrapper = pkgs.callPackage ./nix/ccache.nix {};
+
+  nix-utils = pkgs.fetchFromGitHub {
+    owner = "iknow";
+    repo = "nix-utils";
+    rev = "c13c7a23836c8705452f051d19fc4dff05533b53";
+    sha256 = "0ax7hld5jf132ksdasp80z34dlv75ir0ringzjs15mimrkw8zcac";
+  };
+
+  ociTools = pkgs.callPackage "${nix-utils}/oci" {};
+
+  inherit (zmkPkgs) zmk zephyr;
+
+  accounts = {
+    users.deploy = {
+      uid = 999;
+      group = "deploy";
+      home = "/home/deploy";
+      shell = "/bin/sh";
+    };
+    groups.deploy.gid = 999;
+  };
+
+  baseLayer = {
+    name = "base-layer";
+    path = [ pkgs.busybox ];
+    entries = ociTools.makeFilesystem {
+      inherit accounts;
+      tmp = true;
+      usrBinEnv = "${pkgs.busybox}/bin/env";
+      binSh = "${pkgs.busybox}/bin/sh";
+    };
+  };
+
+  depsLayer = {
+    name = "deps-layer";
+    path = [ pkgs.ccache ];
+    includes = zmk.buildInputs ++ zmk.nativeBuildInputs ++ zmk.zephyrModuleDeps;
+  };
+
+  zmkCompileScript = let
+    zmk' = zmk.override {
+      gcc-arm-embedded = ccacheWrapper.override {
+        unwrappedCC = pkgs.gcc-arm-embedded;
+      };
+    };
+    zmk_glove80_rh = zmk.override { board = "glove80_rh"; };
+    realpath_coreutils = if pkgs.stdenv.isDarwin then pkgs.coreutils else pkgs.busybox;
+  in pkgs.writeShellScriptBin "compileZmk" ''
+    set -eo pipefail
+
+    if [ ! -f "$1" ]; then
+      echo "Usage: compileZmk [file.keymap]" >&2
+      exit 1
+    fi
+
+    KEYMAP="$(${realpath_coreutils}/bin/realpath $1)"
+
+    export PATH=${lib.makeBinPath (with pkgs; zmk'.nativeBuildInputs ++ [ ccache ])}:$PATH
+    export CMAKE_PREFIX_PATH=${zephyr}
+
+    export CCACHE_BASEDIR=$PWD
+    export CCACHE_NOHASHDIR=t
+    export CCACHE_COMPILERCHECK=none
+
+    if [ -n "$DEBUG" ]; then ccache -z; fi
+
+    cmake -G Ninja -S ${zmk'.src}/app ${lib.escapeShellArgs zmk'.cmakeFlags} "-DUSER_CACHE_DIR=/tmp/.cache" "-DKEYMAP_FILE=$KEYMAP" -DBOARD=glove80_lh
+
+    ninja
+
+    if [ -n "$DEBUG" ]; then ccache -s; fi
+
+    cat zephyr/zmk.uf2 ${zmk_glove80_rh}/zmk.uf2 > zephyr/combined.uf2
+  '';
+
+  ccacheCache = pkgs.runCommandNoCC "ccache-cache" {
+    nativeBuildInputs = [ zmkCompileScript ];
+  } ''
+    export CCACHE_DIR=$out
+
+    mkdir /tmp/build
+    cd /tmp/build
+
+    compileZmk ${zmk.src}/app/boards/arm/glove80/glove80.keymap
+  '';
+
+  entrypoint = pkgs.writeShellScriptBin "entrypoint" ''
+    set -euo pipefail
+
+    if [ ! -d "$CCACHE_DIR" ]; then
+      cp -r ${ccacheCache} "$CCACHE_DIR"
+      chmod -R u=rwX,go=u-w "$CCACHE_DIR"
+    fi
+
+    if [ ! -d /tmp/build ]; then
+      mkdir /tmp/build
+    fi
+
+    exec "$@"
+  '';
+
+  startLambda = handler: pkgs.writeShellScriptBin "startLambda" ''
+    set -euo pipefail
+    export PATH=${lib.makeBinPath [ zmkCompileScript ]}:$PATH
+    cd ${lambda.source}
+    ${lambda.bundleEnv}/bin/bundle exec aws_lambda_ric "app.LambdaFunction::${handler}.process"
+  '';
+
+  simulateLambda = lambda: pkgs.writeShellScriptBin "simulateLambda" ''
+    ${pkgs.aws-lambda-rie}/bin/aws-lambda-rie ${lambda}/bin/startLambda
+  '';
+
+  lambdaImage = lambda:
+  let
+    appLayer = {
+      name = "app-layer";
+      path = [ lambda zmkCompileScript ];
+    };
+  in
+  ociTools.makeSimpleImage {
+    name = "zmk-builder-lambda";
+    layers = [ baseLayer depsLayer appLayer ];
+    config = {
+      User = "deploy";
+      WorkingDir = "/tmp";
+      Entrypoint = [ "${entrypoint}/bin/entrypoint" ];
+      Cmd = [ "startLambda" ];
+      Env = [ "CCACHE_DIR=/tmp/ccache" "REVISION=${revision}" ];
+    };
+  };
+
+  # There are two lambda handler functions, depending on whether the lambda is
+  # expected to handle Api Gateway/ELB HTTP requests itself.
+  startHttpLambda = startLambda "HttpHandler";
+  startDirectLambda = startLambda "DirectHandler";
+  httpLambdaImage   = lambdaImage startHttpLambda;
+  directLambdaImage = lambdaImage startDirectLambda;
+
+  simulateDirectLambda = simulateLambda startDirectLambda;
+  simulateHttpLambda   = simulateLambda startHttpLambda;
+in {
+  inherit httpLambdaImage directLambdaImage zmkCompileScript ccacheCache;
+
+  # nix shell -f release.nix simulateDirectLambda -c simulateLambda
+  inherit simulateHttpLambda simulateDirectLambda;
+}