Shard composite image into 10 layers

docker/composite/Dockerfile

@@ -1,4 +1,16 @@
 FROM riju:runtime
 
 COPY docker/composite/install.bash /tmp/
-RUN /tmp/install.bash
+
+# The number of commands here must match NUM_SHARDS in
+# build-composite-image.js.
+RUN /tmp/install.bash 0
+RUN /tmp/install.bash 1
+RUN /tmp/install.bash 2
+RUN /tmp/install.bash 3
+RUN /tmp/install.bash 4
+RUN /tmp/install.bash 5
+RUN /tmp/install.bash 6
+RUN /tmp/install.bash 7
+RUN /tmp/install.bash 8
+RUN /tmp/install.bash 9

docker/composite/install.bash

@@ -2,6 +2,8 @@
 
 set -euxo pipefail
 
+shard="$1"
+
 function riju-curl {
     curl -fsSL "localhost:8487$1"
 }
@@ -17,13 +19,8 @@ export DEBIAN_FRONTEND=noninteractive
 
 apt-get update
 
-riju-curl /shared | while read lang; do
+riju-curl "/shard/${shard}" | while read path; do
-    riju-apt-install "/fs/build/shared/${lang}/riju-shared-${lang}.deb"
+    riju-apt-install "/fs/${path}"
-done
-
-riju-curl /langs | while read lang; do
-    riju-apt-install "/fs/build/lang/${lang}/riju-lang-${lang}.deb"
-    riju-apt-install "/fs/build/config/${lang}/riju-config-${lang}.deb"
 done
 
 rm -rf *.deb

tools/build-composite-image.js

@@ -8,23 +8,50 @@ import { getLocalImageLabel } from "./docker-util.js";
 import { hashDockerfile } from "./hash-dockerfile.js";
 import { runCommand } from "./util.js";
 
+// Number of package installation layers in the composite Docker
+// image. This needs to match the number of installation RUN commands
+// in the composite Dockerfile.
+const NUM_SHARDS = 10;
+
 // Get a Node.js http server object that will serve information and
 // files for packages that should be installed into the composite
 // Docker image.
-function getServer({ langs, sharedDeps }) {
+function getServer({ shards }) {
   const app = express();
-  app.get("/langs", (req, res) => {
+  app.get("/shard/:shard", (req, res) => {
-    res.send(langs.map((lang) => lang + "\n").join(""));
+    res.send(
-  });
+      shards[parseInt(req.params.shard)]
-  app.get("/shared", (req, res) => {
+        .map(({ debPath }) => debPath + "\n")
-    res.send(sharedDeps.map((lang) => lang + "\n").join(""));
+        .join("")
+    );
   });
   app.use("/fs", express.static("."));
   return http.createServer(app);
 }
 
+// Given a list of the packages to be built, split them into shards.
+// Return a list of shards. Each shard is a list of the package
+// objects, such that there are NUM_SHARDS shards. Traversing each
+// shard in order will return the packages in the same order as the
+// original list.
+//
+// Currently this uses an extremely simple algorithm, but that might
+// be improved in the future.
+function getShards(pkgs) {
+  const shards = [];
+  for (let i = 0; i < NUM_SHARDS; ++i) {
+    shards.push([]);
+  }
+  const shardSize = Math.ceil(pkgs.length / NUM_SHARDS);
+  for (let i = 0; i < pkgs.length; ++i) {
+    shards[Math.floor(i / shardSize)].push(pkgs[i]);
+  }
+  return shards;
+}
+
 // Parse command-line arguments, run main functionality, and exit.
 async function main() {
+  const packages = await getPackages();
   const hash = await hashDockerfile(
     "composite",
     {
@@ -37,7 +64,7 @@ async function main() {
       salt: {
         packageHashes: (
           await Promise.all(
-            (await getPackages()).map(async ({ debPath }) => {
+            packages.map(async ({ debPath }) => {
               return (
                 await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
                   getStdout: true,
@@ -50,8 +77,7 @@ async function main() {
     }
   );
   const server = getServer({
-    langs: await getLangs(),
+    shards: getShards(packages),
-    sharedDeps: await getSharedDeps(),
   });
   await new Promise((resolve) => server.listen(8487, "localhost", resolve));
   try {