From feb5aad8f0c6149f0e536cc724e102eab3b088ee Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 18 Mar 2021 17:04:21 -0700 Subject: [PATCH] Build per-language Docker images --- Makefile | 24 +++-- docker/app/Dockerfile | 33 ++++++- .../install.bash => app/install-build.bash} | 4 + docker/app/install.bash | 1 + docker/compile/Dockerfile | 21 ---- docker/composite/Dockerfile | 18 ---- docker/composite/install.bash | 29 ------ docker/lang/Dockerfile | 14 +++ docker/lang/install.bash | 40 ++++++++ docker/runtime/install.bash | 16 +-- tools/build-composite-image.js | 97 ------------------- tools/build-lang-image.js | 73 ++++++++++++++ tools/hash-dockerfile.js | 4 +- tools/util.js | 9 ++ 14 files changed, 197 insertions(+), 186 deletions(-) rename docker/{compile/install.bash => app/install-build.bash} (95%) delete mode 100644 docker/compile/Dockerfile delete mode 100644 docker/composite/Dockerfile delete mode 100755 docker/composite/install.bash create mode 100644 docker/lang/Dockerfile create mode 100755 docker/lang/install.bash delete mode 100644 tools/build-composite-image.js create mode 100644 tools/build-lang-image.js diff --git a/Makefile b/Makefile index 07b1ec9..405ab6e 100644 --- a/Makefile +++ b/Makefile @@ -36,10 +36,11 @@ endif ## Pass NC=1 to disable the Docker cache. Base images are not pulled; ## see 'make pull-base' for that. -image: # I= [NC=1] : Build a Docker image +image: # I= [L=] [NC=1] : Build a Docker image @: $${I} -ifeq ($(I),composite) - node tools/build-composite-image.js +ifeq ($(I),lang) + @: $${L} + node tools/build-lang-image.js --lang $(L) else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else @@ -59,14 +60,23 @@ endif SHELL_ENV := -e Z -e CI -e TEST_PATIENCE -e TEST_CONCURRENCY -shell: # I= [E=1] [P1|P2=] : Launch Docker image with shell +ifeq ($(I),lang) +LANG_TAG := lang-$(L) +else +LANG_TAG := $(I) +endif + +shell: # I= [L=] [E=1] [P1|P2=] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD) -else ifneq (,$(filter $(I),compile app)) +else ifeq ($(I),app) docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) -else ifneq (,$(filter $(I),runtime composite)) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) +else ifneq (,$(filter $(I),runtime lang)) +ifeq ($(I),lang) + @: $${L} +endif + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD) else docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) endif diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index ae4ae8f..04f0cfa 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,12 +1,37 @@ -FROM riju:compile AS compile -FROM riju:composite +FROM ubuntu:rolling AS build +COPY docker/app/install-build.bash /tmp/ +RUN /tmp/install-build.bash + +WORKDIR /src +COPY Makefile ./ + +COPY system ./system/ +RUN make system + +COPY package.json yarn.lock ./ +RUN yarn install + +COPY webpack.config.cjs ./ +COPY frontend/src ./frontend/src/ +RUN make frontend + +COPY frontend/pages ./frontend/pages/ +COPY frontend/styles ./frontend/styles/ +COPY backend ./backend/ + +FROM ubuntu:rolling + +COPY docker/app/install.bash /tmp/ +RUN /tmp/install.bash + +COPY docker/shared/my_init /usr/local/sbin/ ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] + RUN useradd -p '!' -m -l -s /usr/bin/bash riju WORKDIR /src - -COPY --chown=riju:riju --from=compile /src ./ +COPY --chown=riju:riju --from=build /src ./ RUN chown root:riju system/out/*-privileged && chmod a=,g=rx,u=rwxs system/out/*-privileged USER riju diff --git a/docker/compile/install.bash b/docker/app/install-build.bash similarity index 95% rename from docker/compile/install.bash rename to docker/app/install-build.bash index 3bc48a4..ab3363f 100755 --- a/docker/compile/install.bash +++ b/docker/app/install-build.bash @@ -24,3 +24,7 @@ EOF apt-get update apt-get install -y clang g++ make nodejs sudo yarn + +rm -rf /var/lib/apt/lists/* + +rm "$0" diff --git a/docker/app/install.bash b/docker/app/install.bash index abaea1f..d23776a 100755 --- a/docker/app/install.bash +++ b/docker/app/install.bash @@ -6,6 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get dist-upgrade -y + apt-get install -y curl gnupg lsb-release curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - diff --git a/docker/compile/Dockerfile b/docker/compile/Dockerfile deleted file mode 100644 index 43490cc..0000000 --- a/docker/compile/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ubuntu:rolling - -COPY docker/compile/install.bash /tmp/ -RUN /tmp/install.bash - -WORKDIR /src -COPY Makefile ./ - -COPY system ./system/ -RUN make system - -COPY package.json yarn.lock ./ -RUN yarn install - -COPY webpack.config.cjs ./ -COPY frontend/src ./frontend/src/ -RUN make frontend - -COPY frontend/pages ./frontend/pages/ -COPY frontend/styles ./frontend/styles/ -COPY backend ./backend/ diff --git a/docker/composite/Dockerfile b/docker/composite/Dockerfile deleted file mode 100644 index 787dcca..0000000 --- a/docker/composite/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM riju:runtime - -COPY docker/composite/install.bash /tmp/ - -# The number of commands here must match NUM_SHARDS in -# build-composite-image.js. -RUN /tmp/install.bash 0 -RUN /tmp/install.bash 1 -RUN /tmp/install.bash 2 -RUN /tmp/install.bash 3 -RUN /tmp/install.bash 4 -RUN /tmp/install.bash 5 -RUN /tmp/install.bash 6 -RUN /tmp/install.bash 7 -RUN /tmp/install.bash 8 -RUN /tmp/install.bash 9 - -RUN rm /tmp/install.bash diff --git a/docker/composite/install.bash b/docker/composite/install.bash deleted file mode 100755 index 15b0ea6..0000000 --- a/docker/composite/install.bash +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -shard="$1" - -function riju-curl { - curl -fsSL "localhost:8487$1" -} - -function riju-apt-install { - riju-curl "$1" > "$(basename "$1")" - apt-get install -y "./$(basename "$1")" -} - -pushd /tmp - -export DEBIAN_FRONTEND=noninteractive - -apt-get update - -riju-curl "/shard/${shard}" | while read path; do - riju-apt-install "/fs/${path}" -done - -rm -rf *.deb -rm -rf /var/lib/apt/lists/* - -popd diff --git a/docker/lang/Dockerfile b/docker/lang/Dockerfile new file mode 100644 index 0000000..e7504df --- /dev/null +++ b/docker/lang/Dockerfile @@ -0,0 +1,14 @@ +FROM riju:runtime + +ARG LANG + +COPY docker/lang/install.bash /tmp/ +RUN /tmp/install.bash + +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] +RUN rm /usr/local/sbin/pid1.bash + +RUN useradd -p '!' -m -l -s /usr/bin/bash riju +WORKDIR /home/riju/src +USER riju +CMD ["bash"] diff --git a/docker/lang/install.bash b/docker/lang/install.bash new file mode 100755 index 0000000..c0301e7 --- /dev/null +++ b/docker/lang/install.bash @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${LANG}" + +mkdir /tmp/riju-work +pushd /tmp/riju-work + +function riju-curl { + echo >&2 "fetching ./$1" + curl -fsSL "localhost:8487/fs/$1" +} + +export DEBIAN_FRONTEND=noninteractive + +riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" + +( + dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | + (grep -Eo 'riju-shared-[^, ]+' || true) | + sed 's/riju-shared-//' +) | while read name; do + riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb" +done + +if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then + apt-get update +fi + +for file in ./riju-shared-*.deb; do + apt-get install -y "${file}" +done + +apt-get install -y "./riju-lang-${LANG}.deb" + +popd +rm -rf /tmp/riju-work + +rm "$0" diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 06f2fad..b988373 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -90,14 +90,14 @@ EOF # Unfortunately, the Microsoft repo includes a duplicate version of # the libodbc1 package whose version is not in sync with the one # shipped by the corresponding release of Ubuntu. If this one happens -# to be newer, then it'll cause a horrifyingly difficult to diagnose -# error later on while building the composite image because there's a -# conflict between the default-available versions of libodbc1 and -# libodbc1:i386, which surfaces as an inability to install -# dependencies for Erlang. Thanks Microsoft. Please don't. Anyway, -# solution is to pin this repository at a lower priority than the -# Ubuntu standard packages, so the correct version of libodbc1 gets -# installed by default. +# to be newer, then it can cause horrifyingly difficult to diagnose +# errors later on because there's a conflict between the +# default-available versions of libodbc1 and libodbc1:i386, which has +# in the past surfaced as an inability to install dependencies for +# Erlang. Thanks Microsoft. Please don't. Anyway, solution is to pin +# this repository at a lower priority than the Ubuntu standard +# packages, so the correct version of libodbc1 gets installed by +# default. tee -a /etc/apt/preferences.d/riju >/dev/null < { - res.send( - shards[parseInt(req.params.shard)] - .map(({ debPath }) => debPath + "\n") - .join("") - ); - }); - app.use("/fs", express.static(".")); - return http.createServer(app); -} - -// Given a list of the packages to be built, split them into shards. -// Return a list of shards. Each shard is a list of the package -// objects, such that there are NUM_SHARDS shards. Traversing each -// shard in order will return the packages in the same order as the -// original list. -// -// Currently this uses an extremely simple algorithm, but that might -// be improved in the future. -function getShards(pkgs) { - const shards = []; - for (let i = 0; i < NUM_SHARDS; ++i) { - shards.push([]); - } - const shardSize = Math.ceil(pkgs.length / NUM_SHARDS); - for (let i = 0; i < pkgs.length; ++i) { - shards[Math.floor(i / shardSize)].push(pkgs[i]); - } - return shards; -} - -// Parse command-line arguments, run main functionality, and exit. -async function main() { - const packages = await getPackages(); - const hash = await hashDockerfile( - "composite", - { - "riju:runtime": await getLocalImageLabel( - "riju:runtime", - "riju.image-hash" - ), - }, - { - salt: { - packageHashes: ( - await Promise.all( - packages.map(async ({ debPath }) => { - return ( - await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { - getStdout: true, - }) - ).stdout.trim(); - }) - ) - ).sort(), - }, - } - ); - const server = getServer({ - shards: getShards(packages), - }); - await new Promise((resolve) => server.listen(8487, "localhost", resolve)); - try { - await runCommand( - `docker build . -f docker/composite/Dockerfile -t riju:composite` + - ` --network host --no-cache --label riju.image-hash=${hash}` - ); - } finally { - await server.close(); - } - process.exit(0); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js new file mode 100644 index 0000000..e7c8e4b --- /dev/null +++ b/tools/build-lang-image.js @@ -0,0 +1,73 @@ +import { promises as fs } from "fs"; +import http from "http"; + +import { Command } from "commander"; +import express from "express"; + +import { readLangConfig } from "./config.js"; +import { getLocalImageLabel } from "./docker-util.js"; +import { hashDockerfile } from "./hash-dockerfile.js"; +import { getDebHash, runCommand } from "./util.js"; + +// Get a Node.js http server object that will allow the Docker +// build to fetch files from outside the container, without them +// being in the build context. +function getServer() { + const app = express(); + app.use("/fs", express.static(".")); + return http.createServer(app); +} + +// Parse command-line arguments, run main functionality, and exit. +async function main() { + const program = new Command(); + program.requiredOption("--lang ", "language ID"); + program.option("--debug", "interactive debugging"); + program.parse(process.argv); + const { lang, debug } = program; + const hash = await hashDockerfile( + "lang", + { + "riju:runtime": await getLocalImageLabel( + "riju:runtime", + "riju.image-hash" + ), + }, + { + salt: { + langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`), + sharedHashes: ( + await Promise.all( + (((await readLangConfig(lang)).install || {}).riju || []).map( + async (name) => + await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`) + ) + ) + ).sort(), + }, + } + ); + const server = getServer(); + await new Promise((resolve) => server.listen(8487, "localhost", resolve)); + try { + if (debug) { + await runCommand( + `docker run -it --rm -e LANG=${lang} -w /tmp/riju-work --network host riju:runtime` + ); + } else { + await runCommand( + `docker build . -f docker/lang/Dockerfile ` + + `--build-arg LANG=${lang} -t riju:lang-${lang} ` + + `--network host --no-cache --label riju.image-hash=${hash}` + ); + } + } finally { + await server.close(); + } + process.exit(0); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tools/hash-dockerfile.js b/tools/hash-dockerfile.js index 74992d9..bef6be8 100644 --- a/tools/hash-dockerfile.js +++ b/tools/hash-dockerfile.js @@ -187,8 +187,8 @@ async function main() { } const [name] = program.args; const { debug } = program.opts(); - if (name === "composite") { - throw new Error("use build-composite-image.js instead for this"); + if (name === "lang") { + throw new Error("use build-lang-image.js instead for this"); } if (debug) { console.log( diff --git a/tools/util.js b/tools/util.js index 900dc01..dfc29a3 100644 --- a/tools/util.js +++ b/tools/util.js @@ -43,3 +43,12 @@ export async function runCommand(cmd, options) { } return rv; } + +// Return the Riju-Script-Hash field in a .deb file generated by Riju. +export async function getDebHash(debPath) { + return ( + await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { + getStdout: true, + }) + ).stdout.trim(); +}