Build per-language Docker images

Makefile

@@ -36,10 +36,11 @@ endif
 ## Pass NC=1 to disable the Docker cache. Base images are not pulled;
 ## see 'make pull-base' for that.
 
-image: # I=<image> [NC=1] : Build a Docker image
+image: # I=<image> [L=<lang>] [NC=1] : Build a Docker image
 	@: $${I}
-ifeq ($(I),composite)
+ifeq ($(I),lang)
-	node tools/build-composite-image.js
+	@: $${L}
+	node tools/build-lang-image.js --lang $(L)
 else ifneq (,$(filter $(I),admin ci))
 	docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
 else
@@ -59,14 +60,23 @@ endif
 
 SHELL_ENV := -e Z -e CI -e TEST_PATIENCE -e TEST_CONCURRENCY
 
-shell: # I=<shell> [E=1] [P1|P2=<port>] : Launch Docker image with shell
+ifeq ($(I),lang)
+LANG_TAG := lang-$(L)
+else
+LANG_TAG := $(I)
+endif
+
+shell: # I=<shell> [L=<lang>] [E=1] [P1|P2=<port>] : Launch Docker image with shell
 	@: $${I}
 ifneq (,$(filter $(I),admin ci))
 	docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD)
-else ifneq (,$(filter $(I),compile app))
+else ifeq ($(I),app)
 	docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
-else ifneq (,$(filter $(I),runtime composite))
+else ifneq (,$(filter $(I),runtime lang))
-	docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
+ifeq ($(I),lang)
+	@: $${L}
+endif
+	docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD)
 else
 	docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
 endif

docker/app/Dockerfile

@@ -1,12 +1,37 @@
-FROM riju:compile AS compile
+FROM ubuntu:rolling AS build
-FROM riju:composite
 
+COPY docker/app/install-build.bash /tmp/
+RUN /tmp/install-build.bash
+
+WORKDIR /src
+COPY Makefile ./
+
+COPY system ./system/
+RUN make system
+
+COPY package.json yarn.lock ./
+RUN yarn install
+
+COPY webpack.config.cjs ./
+COPY frontend/src ./frontend/src/
+RUN make frontend
+
+COPY frontend/pages ./frontend/pages/
+COPY frontend/styles ./frontend/styles/
+COPY backend ./backend/
+
+FROM ubuntu:rolling
+
+COPY docker/app/install.bash /tmp/
+RUN /tmp/install.bash
+
+COPY docker/shared/my_init /usr/local/sbin/
 ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"]
+
 RUN useradd -p '!' -m -l -s /usr/bin/bash riju
 
 WORKDIR /src
-
+COPY --chown=riju:riju --from=build /src ./
-COPY --chown=riju:riju --from=compile /src ./
 RUN chown root:riju system/out/*-privileged && chmod a=,g=rx,u=rwxs system/out/*-privileged
 
 USER riju

docker/app/install-build.bash

@@ -24,3 +24,7 @@ EOF
 
 apt-get update
 apt-get install -y clang g++ make nodejs sudo yarn
+
+rm -rf /var/lib/apt/lists/*
+
+rm "$0"

docker/app/install.bash

@@ -6,6 +6,7 @@ export DEBIAN_FRONTEND=noninteractive
 
 apt-get update
 apt-get dist-upgrade -y
+
 apt-get install -y curl gnupg lsb-release
 
 curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -

docker/compile/Dockerfile

@@ -1,21 +0,0 @@
-FROM ubuntu:rolling
-
-COPY docker/compile/install.bash /tmp/
-RUN /tmp/install.bash
-
-WORKDIR /src
-COPY Makefile ./
-
-COPY system ./system/
-RUN make system
-
-COPY package.json yarn.lock ./
-RUN yarn install
-
-COPY webpack.config.cjs ./
-COPY frontend/src ./frontend/src/
-RUN make frontend
-
-COPY frontend/pages ./frontend/pages/
-COPY frontend/styles ./frontend/styles/
-COPY backend ./backend/

docker/composite/Dockerfile

@@ -1,18 +0,0 @@
-FROM riju:runtime
-
-COPY docker/composite/install.bash /tmp/
-
-# The number of commands here must match NUM_SHARDS in
-# build-composite-image.js.
-RUN /tmp/install.bash 0
-RUN /tmp/install.bash 1
-RUN /tmp/install.bash 2
-RUN /tmp/install.bash 3
-RUN /tmp/install.bash 4
-RUN /tmp/install.bash 5
-RUN /tmp/install.bash 6
-RUN /tmp/install.bash 7
-RUN /tmp/install.bash 8
-RUN /tmp/install.bash 9
-
-RUN rm /tmp/install.bash

docker/composite/install.bash

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-set -euxo pipefail
-
-shard="$1"
-
-function riju-curl {
-    curl -fsSL "localhost:8487$1"
-}
-
-function riju-apt-install {
-    riju-curl "$1" > "$(basename "$1")"
-    apt-get install -y "./$(basename "$1")"
-}
-
-pushd /tmp
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-
-riju-curl "/shard/${shard}" | while read path; do
-    riju-apt-install "/fs/${path}"
-done
-
-rm -rf *.deb
-rm -rf /var/lib/apt/lists/*
-
-popd

docker/lang/Dockerfile

@@ -0,0 +1,14 @@
+FROM riju:runtime
+
+ARG LANG
+
+COPY docker/lang/install.bash /tmp/
+RUN /tmp/install.bash
+
+ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"]
+RUN rm /usr/local/sbin/pid1.bash
+
+RUN useradd -p '!' -m -l -s /usr/bin/bash riju
+WORKDIR /home/riju/src
+USER riju
+CMD ["bash"]

docker/lang/install.bash

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${LANG}"
+
+mkdir /tmp/riju-work
+pushd /tmp/riju-work
+
+function riju-curl {
+    echo >&2 "fetching ./$1"
+    curl -fsSL "localhost:8487/fs/$1"
+}
+
+export DEBIAN_FRONTEND=noninteractive
+
+riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb"
+
+(
+    dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends |
+        (grep -Eo 'riju-shared-[^, ]+' || true) |
+        sed 's/riju-shared-//'
+) | while read name; do
+    riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb"
+done
+
+if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then
+    apt-get update
+fi
+
+for file in ./riju-shared-*.deb; do
+    apt-get install -y "${file}"
+done
+
+apt-get install -y "./riju-lang-${LANG}.deb"
+
+popd
+rm -rf /tmp/riju-work
+
+rm "$0"

docker/runtime/install.bash

@@ -90,14 +90,14 @@ EOF
 # Unfortunately, the Microsoft repo includes a duplicate version of
 # the libodbc1 package whose version is not in sync with the one
 # shipped by the corresponding release of Ubuntu. If this one happens
-# to be newer, then it'll cause a horrifyingly difficult to diagnose
+# to be newer, then it can cause horrifyingly difficult to diagnose
-# error later on while building the composite image because there's a
+# errors later on because there's a conflict between the
-# conflict between the default-available versions of libodbc1 and
+# default-available versions of libodbc1 and libodbc1:i386, which has
-# libodbc1:i386, which surfaces as an inability to install
+# in the past surfaced as an inability to install dependencies for
-# dependencies for Erlang. Thanks Microsoft. Please don't. Anyway,
+# Erlang. Thanks Microsoft. Please don't. Anyway, solution is to pin
-# solution is to pin this repository at a lower priority than the
+# this repository at a lower priority than the Ubuntu standard
-# Ubuntu standard packages, so the correct version of libodbc1 gets
+# packages, so the correct version of libodbc1 gets installed by
-# installed by default.
+# default.
 tee -a /etc/apt/preferences.d/riju >/dev/null <<EOF
 Package: *
 Pin: origin packages.microsoft.com

tools/build-composite-image.js

@@ -1,97 +0,0 @@
-import { promises as fs } from "fs";
-import http from "http";
-
-import express from "express";
-
-import { getLangs, getPackages, getSharedDeps } from "./config.js";
-import { getLocalImageLabel } from "./docker-util.js";
-import { hashDockerfile } from "./hash-dockerfile.js";
-import { runCommand } from "./util.js";
-
-// Number of package installation layers in the composite Docker
-// image. This needs to match the number of installation RUN commands
-// in the composite Dockerfile.
-const NUM_SHARDS = 10;
-
-// Get a Node.js http server object that will serve information and
-// files for packages that should be installed into the composite
-// Docker image.
-function getServer({ shards }) {
-  const app = express();
-  app.get("/shard/:shard", (req, res) => {
-    res.send(
-      shards[parseInt(req.params.shard)]
-        .map(({ debPath }) => debPath + "\n")
-        .join("")
-    );
-  });
-  app.use("/fs", express.static("."));
-  return http.createServer(app);
-}
-
-// Given a list of the packages to be built, split them into shards.
-// Return a list of shards. Each shard is a list of the package
-// objects, such that there are NUM_SHARDS shards. Traversing each
-// shard in order will return the packages in the same order as the
-// original list.
-//
-// Currently this uses an extremely simple algorithm, but that might
-// be improved in the future.
-function getShards(pkgs) {
-  const shards = [];
-  for (let i = 0; i < NUM_SHARDS; ++i) {
-    shards.push([]);
-  }
-  const shardSize = Math.ceil(pkgs.length / NUM_SHARDS);
-  for (let i = 0; i < pkgs.length; ++i) {
-    shards[Math.floor(i / shardSize)].push(pkgs[i]);
-  }
-  return shards;
-}
-
-// Parse command-line arguments, run main functionality, and exit.
-async function main() {
-  const packages = await getPackages();
-  const hash = await hashDockerfile(
-    "composite",
-    {
-      "riju:runtime": await getLocalImageLabel(
-        "riju:runtime",
-        "riju.image-hash"
-      ),
-    },
-    {
-      salt: {
-        packageHashes: (
-          await Promise.all(
-            packages.map(async ({ debPath }) => {
-              return (
-                await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
-                  getStdout: true,
-                })
-              ).stdout.trim();
-            })
-          )
-        ).sort(),
-      },
-    }
-  );
-  const server = getServer({
-    shards: getShards(packages),
-  });
-  await new Promise((resolve) => server.listen(8487, "localhost", resolve));
-  try {
-    await runCommand(
-      `docker build . -f docker/composite/Dockerfile -t riju:composite` +
-        ` --network host --no-cache --label riju.image-hash=${hash}`
-    );
-  } finally {
-    await server.close();
-  }
-  process.exit(0);
-}
-
-main().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});

tools/build-lang-image.js

@@ -0,0 +1,73 @@
+import { promises as fs } from "fs";
+import http from "http";
+
+import { Command } from "commander";
+import express from "express";
+
+import { readLangConfig } from "./config.js";
+import { getLocalImageLabel } from "./docker-util.js";
+import { hashDockerfile } from "./hash-dockerfile.js";
+import { getDebHash, runCommand } from "./util.js";
+
+// Get a Node.js http server object that will allow the Docker
+// build to fetch files from outside the container, without them
+// being in the build context.
+function getServer() {
+  const app = express();
+  app.use("/fs", express.static("."));
+  return http.createServer(app);
+}
+
+// Parse command-line arguments, run main functionality, and exit.
+async function main() {
+  const program = new Command();
+  program.requiredOption("--lang <id>", "language ID");
+  program.option("--debug", "interactive debugging");
+  program.parse(process.argv);
+  const { lang, debug } = program;
+  const hash = await hashDockerfile(
+    "lang",
+    {
+      "riju:runtime": await getLocalImageLabel(
+        "riju:runtime",
+        "riju.image-hash"
+      ),
+    },
+    {
+      salt: {
+        langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`),
+        sharedHashes: (
+          await Promise.all(
+            (((await readLangConfig(lang)).install || {}).riju || []).map(
+              async (name) =>
+                await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`)
+            )
+          )
+        ).sort(),
+      },
+    }
+  );
+  const server = getServer();
+  await new Promise((resolve) => server.listen(8487, "localhost", resolve));
+  try {
+    if (debug) {
+      await runCommand(
+        `docker run -it --rm -e LANG=${lang} -w /tmp/riju-work --network host riju:runtime`
+      );
+    } else {
+      await runCommand(
+        `docker build . -f docker/lang/Dockerfile ` +
+          `--build-arg LANG=${lang} -t riju:lang-${lang} ` +
+          `--network host --no-cache --label riju.image-hash=${hash}`
+      );
+    }
+  } finally {
+    await server.close();
+  }
+  process.exit(0);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

tools/hash-dockerfile.js

@@ -187,8 +187,8 @@ async function main() {
   }
   const [name] = program.args;
   const { debug } = program.opts();
-  if (name === "composite") {
+  if (name === "lang") {
-    throw new Error("use build-composite-image.js instead for this");
+    throw new Error("use build-lang-image.js instead for this");
   }
   if (debug) {
     console.log(

tools/util.js

@@ -43,3 +43,12 @@ export async function runCommand(cmd, options) {
   }
   return rv;
 }
+
+// Return the Riju-Script-Hash field in a .deb file generated by Riju.
+export async function getDebHash(debPath) {
+  return (
+    await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
+      getStdout: true,
+    })
+  ).stdout.trim();
+}