From c7fc3432cd6fa9b09fc7de7481960f57a5315c1d Mon Sep 17 00:00:00 2001 From: Chris Andreae Date: Mon, 11 Oct 2021 16:49:33 +0900 Subject: [PATCH] Docker container and lambda function for performing firmware builds Provides an entry point that builds and returns a combined LH + RH keyboard firmware when provided a keymap via a POST body. Wraps compilation with ccache, and includes a pre-warmed cache of the build in /tmp/ccache. To maximize chance of a direct cache hit, changes the lambda driver to always build in /tmp/build. some back of the envelope measurements (2012 xeon e3-1230v2, nixos) clean build, no cache -> 21.308 clean build, cache -> 7.145 modified keymap, clean build, cache -> 12.127 --- .github/workflows/build-container.yml | 72 +++++++++++ .github/workflows/cleanup-container.yml | 38 ++++++ .github/workflows/nix-build.yml | 2 +- lambda/Gemfile | 7 ++ lambda/Gemfile.lock | 36 ++++++ lambda/app.rb | 58 +++++++++ lambda/compiler.rb | 59 +++++++++ lambda/default.nix | 26 ++++ lambda/gemset.nix | 88 ++++++++++++++ lambda/shell.nix | 9 ++ lambda/web_app.rb | 43 +++++++ nix/ccache.nix | 43 +++++++ nix/zmk.nix | 4 +- release.nix | 153 ++++++++++++++++++++++++ 14 files changed, 636 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-container.yml create mode 100644 .github/workflows/cleanup-container.yml create mode 100644 lambda/Gemfile create mode 100644 lambda/Gemfile.lock create mode 100644 lambda/app.rb create mode 100644 lambda/compiler.rb create mode 100644 lambda/default.nix create mode 100644 lambda/gemset.nix create mode 100644 lambda/shell.nix create mode 100644 lambda/web_app.rb create mode 100644 nix/ccache.nix create mode 100644 release.nix diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 00000000..70f852b1 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,72 @@ +name: Build Compiler Service Container + +on: + push: + branches: + - 'main' + tags: + - '*' + pull_request: + branches: + - main + +jobs: + build: + if: github.repository == 'moergo-sc/zmk' + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + env: + ECR_REPOSITORY: zmk-builder-lambda + UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV + REVISION_TAG: ${{ github.sha }} + steps: + - uses: actions/checkout@v2.4.0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder + aws-region: us-east-1 + - name: Extract image tag name + shell: bash + run: | + if [ "$GITHUB_REF" = "refs/heads/main" ]; then + tag="latest" + elif [ "$GITHUB_HEAD_REF" ]; then + pr=${GITHUB_REF#refs/pull/} + pr=${pr%/merge} + tag="pr${pr}.${GITHUB_HEAD_REF}" + elif [[ "$GITHUB_REF" == refs/tags/* ]]; then + tag="${GITHUB_REF#refs/tags/}" + else + echo "Not a release branch or tag" >&2 + exit 1 + fi + # Replace / with . in container tag names + tag="${tag//\//.}" + echo "IMAGE_TAG=${tag}" >> $GITHUB_ENV + id: extract_name + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - uses: cachix/install-nix-action@v16 + with: + nix_path: nixpkgs=channel:nixos-22.05 + - uses: cachix/cachix-action@v12 + with: + name: moergo-glove80-zmk-dev + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - name: Build lambda image + run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A directLambdaImage -o directLambdaImage + - name: Import OCI image into docker-daemon + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: skopeo --insecure-policy copy oci:directLambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + - name: Push container image to Amazon ECR + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: docker push $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + - name: Notify the build pipeline that the compile containers have updated + run: aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION /dev/null diff --git a/.github/workflows/cleanup-container.yml b/.github/workflows/cleanup-container.yml new file mode 100644 index 00000000..e111a496 --- /dev/null +++ b/.github/workflows/cleanup-container.yml @@ -0,0 +1,38 @@ +name: Clean up PR Compiler Service Container + +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + build: + if: github.repository == 'moergo-sc/zmk' + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + env: + ECR_REPOSITORY: zmk-builder-lambda + UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV + PR_NUMBER: ${{ github.event.number }} + steps: + - name: Extract image tag name + shell: bash + run: | + tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}" + # Replace / with . in container tag names + tag="${tag//\//.}" + echo "IMAGE_TAG=${tag}" >> $GITHUB_ENV + id: extract_name + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder + aws-region: us-east-1 + - name: Delete the container image for the PR from ECR + run: aws ecr batch-delete-image --repository-name $ECR_REPOSITORY --image-ids imageTag=$IMAGE_TAG + - name: Notify the build pipeline that the compile containers have updated + run: aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION /dev/null diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 82a9da47..564c803b 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -1,4 +1,4 @@ -name: Build +name: Build Glove80 Firmware on: push: diff --git a/lambda/Gemfile b/lambda/Gemfile new file mode 100644 index 00000000..dd1dca60 --- /dev/null +++ b/lambda/Gemfile @@ -0,0 +1,7 @@ +source 'https://rubygems.org' +gem 'aws_lambda_ric' +gem 'rack' +gem 'sinatra', '~> 2' + +# The version on rubygems (1.0.7) is very out of date +gem 'serverless-rack', git: 'https://github.com/logandk/serverless-rack', branch: '7364305bc' diff --git a/lambda/Gemfile.lock b/lambda/Gemfile.lock new file mode 100644 index 00000000..0c2f9299 --- /dev/null +++ b/lambda/Gemfile.lock @@ -0,0 +1,36 @@ +GIT + remote: https://github.com/logandk/serverless-rack + revision: 7364305bcbbf7f6cc6851497069a5a4cb91936b1 + branch: 7364305bc + specs: + serverless-rack (1.0.7) + rack (~> 2.0) + +GEM + remote: https://rubygems.org/ + specs: + aws_lambda_ric (2.0.0) + mustermann (2.0.2) + ruby2_keywords (~> 0.0.1) + rack (2.2.4) + rack-protection (2.2.2) + rack + ruby2_keywords (0.0.5) + sinatra (2.2.2) + mustermann (~> 2.0) + rack (~> 2.2) + rack-protection (= 2.2.2) + tilt (~> 2.0) + tilt (2.0.11) + +PLATFORMS + ruby + +DEPENDENCIES + aws_lambda_ric + rack + serverless-rack! + sinatra (~> 2) + +BUNDLED WITH + 2.1.4 diff --git a/lambda/app.rb b/lambda/app.rb new file mode 100644 index 00000000..8c7a6973 --- /dev/null +++ b/lambda/app.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'rack' +require 'serverless_rack' + +require './web_app' +require './compiler' + +$app = Rack::Builder.new do + run WebApp +end.to_app + +module LambdaFunction + # Handle a API Gateway/ALB-structured HTTP request using the Sinatra app + class HttpHandler + def self.process(event:, context:) + handle_request(app: $app, event: event, context: context) + end + end + + # Handle a non-HTTP proxied request, returning either the compiled result or + # an error as JSON. + class DirectHandler + REVISION = ENV.fetch('REVISION', 'unknown') + + def self.process(event:, context:) + return { type: 'keep_alive' } if event.has_key?('keep_alive') + + keymap_data = event.fetch('keymap') do + return error(status: 400, message: 'Missing required argument: keymap') + end + + keymap_data = + begin + Base64.strict_decode64(keymap_data) + rescue ArgumentError + return error(status: 400, message: 'Invalid Base64 in keymap input') + end + + result, log = + begin + Compiler.new.compile(keymap_data) + rescue Compiler::CompileError => e + return error(status: e.status, message: e.message, detail: e.log) + end + + result = Base64.strict_encode64(result) + + { type: 'result', result: result, log: log, revision: REVISION } + rescue StandardError => e + error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message]) + end + + def self.error(status:, message:, detail: nil) + { type: 'error', status: status, message: message, detail: detail, revision: REVISION } + end + end +end diff --git a/lambda/compiler.rb b/lambda/compiler.rb new file mode 100644 index 00000000..94fc7230 --- /dev/null +++ b/lambda/compiler.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'tmpdir' +require 'json' +require 'base64' + +class Compiler + class CompileError < RuntimeError + attr_reader :status, :log + + def initialize(message, status: 400, log:) + super(message) + @status = status + @log = log + end + end + + def compile(keymap_data) + in_build_dir do + File.open('build.keymap', 'w') do |io| + io.write(keymap_data) + end + + compile_output = nil + + IO.popen(['compileZmk', './build.keymap'], err: [:child, :out]) do |io| + compile_output = io.read + end + + compile_output = compile_output.split("\n") + + unless $?.success? + status = $?.exitstatus + raise CompileError.new("Compile failed with exit status #{status}", log: compile_output) + end + + unless File.exist?('zephyr/combined.uf2') + raise CompileError.new('Compile failed to produce result binary', status: 500, log: compile_output) + end + + result = File.read('zephyr/combined.uf2') + + [result, compile_output] + end + end + + # Lambda is single-process per container, and we get substantial speedups + # from ccache by always building in the same path + BUILD_DIR = '/tmp/build' + + def in_build_dir + FileUtils.remove_entry(BUILD_DIR, true) + Dir.mkdir(BUILD_DIR) + Dir.chdir(BUILD_DIR) + yield + ensure + FileUtils.remove_entry(BUILD_DIR, true) rescue nil + end +end diff --git a/lambda/default.nix b/lambda/default.nix new file mode 100644 index 00000000..6a34b0d1 --- /dev/null +++ b/lambda/default.nix @@ -0,0 +1,26 @@ +{ pkgs ? import {} }: + +with pkgs; + +let + bundleEnv = bundlerEnv { + name = "lambda-bundler-env"; + ruby = ruby_3_1; + gemfile = ./Gemfile; + lockfile = ./Gemfile.lock; + gemset = ./gemset.nix; + }; + + source = stdenv.mkDerivation { + name = "lambda-builder"; + version = "0.0.1"; + src = ./.; + installPhase = '' + cp -r ./ $out + ''; + }; + +in +{ + inherit bundleEnv source; +} diff --git a/lambda/gemset.nix b/lambda/gemset.nix new file mode 100644 index 00000000..11b36ff8 --- /dev/null +++ b/lambda/gemset.nix @@ -0,0 +1,88 @@ +{ + aws_lambda_ric = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "19c4xlgnhgwf3n3z57z16nmr76jd2vihhshknm5zqip2g00awhi1"; + type = "gem"; + }; + version = "2.0.0"; + }; + mustermann = { + dependencies = ["ruby2_keywords"]; + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0m70qz27mlv2rhk4j1li6pw797gmiwwqg02vcgxcxr1rq2v53rnb"; + type = "gem"; + }; + version = "2.0.2"; + }; + rack = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0axc6w0rs4yj0pksfll1hjgw1k6a5q0xi2lckh91knfb72v348pa"; + type = "gem"; + }; + version = "2.2.4"; + }; + rack-protection = { + dependencies = ["rack"]; + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "169jzzgvbjrqmz4q55wp9pg4ji2h90mggcdxy152gv5vp96l2hgx"; + type = "gem"; + }; + version = "2.2.2"; + }; + ruby2_keywords = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1vz322p8n39hz3b4a9gkmz9y7a5jaz41zrm2ywf31dvkqm03glgz"; + type = "gem"; + }; + version = "0.0.5"; + }; + serverless-rack = { + dependencies = ["rack"]; + groups = ["default"]; + platforms = []; + source = { + fetchSubmodules = false; + rev = "7364305bcbbf7f6cc6851497069a5a4cb91936b1"; + sha256 = "0c7ch0s0nl70p6ijg7q0jnq8ca2rhp5wqfp91kai81dy7d71mq65"; + type = "git"; + url = "https://github.com/logandk/serverless-rack"; + }; + version = "1.0.7"; + }; + sinatra = { + dependencies = ["mustermann" "rack" "rack-protection" "tilt"]; + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0mbjp75dy35q796iard8izsy7gk55g2c3q864r2p13my3yjmlcvz"; + type = "gem"; + }; + version = "2.2.2"; + }; + tilt = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "186nfbcsk0l4l86gvng1fw6jq6p6s7rc0caxr23b3pnbfb20y63v"; + type = "gem"; + }; + version = "2.0.11"; + }; +} diff --git a/lambda/shell.nix b/lambda/shell.nix new file mode 100644 index 00000000..2f1eca8b --- /dev/null +++ b/lambda/shell.nix @@ -0,0 +1,9 @@ +{ pkgs ? (import {})}: + +let + lambda = import ./default.nix { inherit pkgs; }; +in +pkgs.stdenv.mkDerivation { + name = "lambda-shell"; + buildInputs = [lambda.bundleEnv.wrappedRuby]; +} diff --git a/lambda/web_app.rb b/lambda/web_app.rb new file mode 100644 index 00000000..6af1b0c1 --- /dev/null +++ b/lambda/web_app.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'sinatra/base' +require './compiler' + +class WebApp < Sinatra::Base + set :environment, :production + set :show_exceptions, false + set :logging, nil + set :default_content_type, 'application/json' + + def json_body(hash) + body(hash.to_json) + end + + post '/api/compile' do + request.body.rewind + keymap_data = request.body.read + result, log = Compiler.new.compile(keymap_data) + + status 200 + content_type 'application/octet-stream' + headers 'X-Debug-Output': log.to_json + body result + end + + error Compiler::CompileError do + e = env['sinatra.error'] + status e.status + json_body(error: e.message, detail: e.log) + end + + error do + e = env['sinatra.error'] + status 500 + json_body(error: "Unexpected error: #{e.class}", detail: [e.message]) + end + + not_found do + status 404 + json_body(error: 'No such path', detail: nil) + end +end diff --git a/nix/ccache.nix b/nix/ccache.nix new file mode 100644 index 00000000..03015314 --- /dev/null +++ b/nix/ccache.nix @@ -0,0 +1,43 @@ +{ stdenv, lib, makeWrapper, ccache +, unwrappedCC ? stdenv.cc.cc, extraConfig ? "" }: + +# copied from ccache in nixpkgs, modified to glob over prefixes. Also doesn't +# pass lib. Why was it passing lib? +stdenv.mkDerivation { + name = "ccache-links"; + passthru = { + isClang = unwrappedCC.isClang or false; + isGNU = unwrappedCC.isGNU or false; + }; + nativeBuildInputs = [ makeWrapper ]; + buildCommand = '' + mkdir -p $out/bin + + wrap() { + local cname="$(basename $1)" + if [ -x "${unwrappedCC}/bin/$cname" ]; then + echo "Wrapping $1" + makeWrapper ${ccache}/bin/ccache $out/bin/$cname \ + --run ${lib.escapeShellArg extraConfig} \ + --add-flags ${unwrappedCC}/bin/$cname + fi + } + + wrapAll() { + for prog in "$@"; do + wrap "$prog" + done + } + + wrapAll ${unwrappedCC}/bin/{*cc,*c++,*gcc,*g++,*clang,*clang++} + + for executable in $(ls ${unwrappedCC}/bin); do + if [ ! -x "$out/bin/$executable" ]; then + ln -s ${unwrappedCC}/bin/$executable $out/bin/$executable + fi + done + for file in $(ls ${unwrappedCC} | grep -vw bin); do + ln -s ${unwrappedCC}/$file $out/$file + done + ''; +} diff --git a/nix/zmk.nix b/nix/zmk.nix index ec6b9f22..37144fe0 100644 --- a/nix/zmk.nix +++ b/nix/zmk.nix @@ -66,7 +66,9 @@ stdenvNoCC.mkDerivation { # Transient state relPath == "build" || relPath == ".west" || # Fetched by west - relPath == "modules" || relPath == "tools" || relPath == "zephyr" + relPath == "modules" || relPath == "tools" || relPath == "zephyr" || + # Not part of ZMK + relPath == "lambda" || relPath == ".github" ); }; diff --git a/release.nix b/release.nix new file mode 100644 index 00000000..26233b30 --- /dev/null +++ b/release.nix @@ -0,0 +1,153 @@ +{ pkgs ? (import ./nix/pinned-nixpkgs.nix {}), revision ? "HEAD" }: + +let + lib = pkgs.lib; + zmkPkgs = (import ./default.nix { inherit pkgs; }); + lambda = (import ./lambda { inherit pkgs; }); + ccacheWrapper = pkgs.callPackage ./nix/ccache.nix {}; + + nix-utils = pkgs.fetchFromGitHub { + owner = "iknow"; + repo = "nix-utils"; + rev = "c13c7a23836c8705452f051d19fc4dff05533b53"; + sha256 = "0ax7hld5jf132ksdasp80z34dlv75ir0ringzjs15mimrkw8zcac"; + }; + + ociTools = pkgs.callPackage "${nix-utils}/oci" {}; + + inherit (zmkPkgs) zmk zephyr; + + accounts = { + users.deploy = { + uid = 999; + group = "deploy"; + home = "/home/deploy"; + shell = "/bin/sh"; + }; + groups.deploy.gid = 999; + }; + + baseLayer = { + name = "base-layer"; + path = [ pkgs.busybox ]; + entries = ociTools.makeFilesystem { + inherit accounts; + tmp = true; + usrBinEnv = "${pkgs.busybox}/bin/env"; + binSh = "${pkgs.busybox}/bin/sh"; + }; + }; + + depsLayer = { + name = "deps-layer"; + path = [ pkgs.ccache ]; + includes = zmk.buildInputs ++ zmk.nativeBuildInputs ++ zmk.zephyrModuleDeps; + }; + + zmkCompileScript = let + zmk' = zmk.override { + gcc-arm-embedded = ccacheWrapper.override { + unwrappedCC = pkgs.gcc-arm-embedded; + }; + }; + zmk_glove80_rh = zmk.override { board = "glove80_rh"; }; + realpath_coreutils = if pkgs.stdenv.isDarwin then pkgs.coreutils else pkgs.busybox; + in pkgs.writeShellScriptBin "compileZmk" '' + set -eo pipefail + + if [ ! -f "$1" ]; then + echo "Usage: compileZmk [file.keymap]" >&2 + exit 1 + fi + + KEYMAP="$(${realpath_coreutils}/bin/realpath $1)" + + export PATH=${lib.makeBinPath (with pkgs; zmk'.nativeBuildInputs ++ [ ccache ])}:$PATH + export CMAKE_PREFIX_PATH=${zephyr} + + export CCACHE_BASEDIR=$PWD + export CCACHE_NOHASHDIR=t + export CCACHE_COMPILERCHECK=none + + if [ -n "$DEBUG" ]; then ccache -z; fi + + cmake -G Ninja -S ${zmk'.src}/app ${lib.escapeShellArgs zmk'.cmakeFlags} "-DUSER_CACHE_DIR=/tmp/.cache" "-DKEYMAP_FILE=$KEYMAP" -DBOARD=glove80_lh + + ninja + + if [ -n "$DEBUG" ]; then ccache -s; fi + + cat zephyr/zmk.uf2 ${zmk_glove80_rh}/zmk.uf2 > zephyr/combined.uf2 + ''; + + ccacheCache = pkgs.runCommandNoCC "ccache-cache" { + nativeBuildInputs = [ zmkCompileScript ]; + } '' + export CCACHE_DIR=$out + + mkdir /tmp/build + cd /tmp/build + + compileZmk ${zmk.src}/app/boards/arm/glove80/glove80.keymap + ''; + + entrypoint = pkgs.writeShellScriptBin "entrypoint" '' + set -euo pipefail + + if [ ! -d "$CCACHE_DIR" ]; then + cp -r ${ccacheCache} "$CCACHE_DIR" + chmod -R u=rwX,go=u-w "$CCACHE_DIR" + fi + + if [ ! -d /tmp/build ]; then + mkdir /tmp/build + fi + + exec "$@" + ''; + + startLambda = handler: pkgs.writeShellScriptBin "startLambda" '' + set -euo pipefail + export PATH=${lib.makeBinPath [ zmkCompileScript ]}:$PATH + cd ${lambda.source} + ${lambda.bundleEnv}/bin/bundle exec aws_lambda_ric "app.LambdaFunction::${handler}.process" + ''; + + simulateLambda = lambda: pkgs.writeShellScriptBin "simulateLambda" '' + ${pkgs.aws-lambda-rie}/bin/aws-lambda-rie ${lambda}/bin/startLambda + ''; + + lambdaImage = lambda: + let + appLayer = { + name = "app-layer"; + path = [ lambda zmkCompileScript ]; + }; + in + ociTools.makeSimpleImage { + name = "zmk-builder-lambda"; + layers = [ baseLayer depsLayer appLayer ]; + config = { + User = "deploy"; + WorkingDir = "/tmp"; + Entrypoint = [ "${entrypoint}/bin/entrypoint" ]; + Cmd = [ "startLambda" ]; + Env = [ "CCACHE_DIR=/tmp/ccache" "REVISION=${revision}" ]; + }; + }; + + # There are two lambda handler functions, depending on whether the lambda is + # expected to handle Api Gateway/ELB HTTP requests itself. + startHttpLambda = startLambda "HttpHandler"; + startDirectLambda = startLambda "DirectHandler"; + httpLambdaImage = lambdaImage startHttpLambda; + directLambdaImage = lambdaImage startDirectLambda; + + simulateDirectLambda = simulateLambda startDirectLambda; + simulateHttpLambda = simulateLambda startHttpLambda; +in { + inherit httpLambdaImage directLambdaImage zmkCompileScript ccacheCache; + + # nix shell -f release.nix simulateDirectLambda -c simulateLambda + inherit simulateHttpLambda simulateDirectLambda; +}