Make CI run faster by fixing bugs

Makefile

@@ -67,7 +67,7 @@ shell:
 	@: $${I}
 ifeq ($(I),admin)
 	docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) --network host riju:$(I) $(CMD)
-else ifeq ($(I),compile)
+else ifneq (,$(filter $(I),compile app))
 	docker run -it --rm --hostname $(I) $(SHELL_PORTS) riju:$(I) $(CMD)
 else
 	docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) riju:$(I) $(CMD)

bin/skopeo

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ "${OSTYPE:-}" != darwin* ]] && [[ "${EUID}" != 0 ]]; then
+    exec sudo -E skopeo "$@"
+else
+    exec skopeo "$@"
+fi

docker/admin/install.bash

@@ -39,6 +39,7 @@ make
 man
 nodejs
 packer
+skopeo
 ssh
 sudo
 tmux

tools/build-composite-image.js

@@ -4,6 +4,7 @@ import http from "http";
 import express from "express";
 
 import { getLangs } from "./config.js";
+import { hashCompositeImage } from "./hash-composite-image.js";
 import { runCommand } from "./util.js";
 
 // Get a Node.js http server object that will serve information and
@@ -23,8 +24,10 @@ async function main() {
   const server = getServer(await getLangs());
   await new Promise((resolve) => server.listen(8487, "localhost", resolve));
   try {
+    const hash = await hashCompositeImage("debs");
     await runCommand(
-      "docker build . -f docker/composite/Dockerfile -t riju:composite --network host --no-cache"
+      `docker build . -f docker/composite/Dockerfile -t riju:composite` +
+        ` --network host --no-cache --label riju-composite-hash=${hash}`
     );
   } finally {
     await server.close();

tools/hash-composite-image.js

@@ -1,20 +1,28 @@
 import crypto from "crypto";
 import { promises as fs } from "fs";
 import process from "process";
+import url from "url";
 
 import { getLangs } from "./config.js";
 import { runCommand } from "./util.js";
 
-// Parse command-line arguments, run main functionality, and exit.
-async function main() {
-  const args = process.argv.slice(2);
-  if (args.length !== 1) {
-    console.error(
-      "usage: node hash-composite-image.js (scripts | debs | remote)"
-    );
-    process.exit(1);
-  }
-  const mode = args[0];
+// Return the composite image hash as a string. This is designed for
+// library usage; main() just calls it and prints the result.
+//
+// If mode is "scripts" then each build script is run through SHA-1
+// and the resulting hashes are hashed together.
+//
+// If mode is "debs" then the Riju-Script-Hash value written into the
+// metadata of each .deb (all of them must exist locally) is
+// extracted, and they are all hashed together.
+//
+// If mode is "s3" then all the published hashes are retrieved from
+// S3. The relevant ones are hashed together.
+//
+// If mode is "registry" then the composite Docker image published to
+// ${DOCKER_REPO} is inspected to extract its composite hash from an
+// image label.
+export async function hashCompositeImage(mode) {
   let getHash;
   switch (mode) {
     case "scripts":
@@ -50,6 +58,24 @@ async function main() {
       );
       getHash = async (lang, type) => remoteHashes[`riju-${type}-${lang}`];
       break;
+    case "registry":
+      const tags = (
+        await runCommand(
+          `skopeo list-tags "docker://\${DOCKER_REPO}" | jq -r '.Tags[]'`,
+          { getStdout: true }
+        )
+      ).stdout
+        .trim()
+        .split("\n");
+      if (!tags.includes("composite")) {
+        return "not yet published";
+      }
+      return (
+        await runCommand(
+          `skopeo inspect docker://raxod502/riju:composite | jq -r '.Labels["riju-composite-hash"]'`,
+          { getStdout: true }
+        )
+      ).stdout.trim();
     default:
       console.error(`hash-composite-image.js: unsupported mode: ${mode}`);
       process.exit(1);
@@ -66,11 +92,26 @@ async function main() {
     }
   }
   const allHashes = Object.values(hashes).sort().join(",");
-  console.log(crypto.createHash("sha1").update(allHashes).digest("hex"));
+  return crypto.createHash("sha1").update(allHashes).digest("hex");
+}
+
+// Parse command-line arguments, run main functionality, and exit.
+async function main() {
+  const args = process.argv.slice(2);
+  if (args.length !== 1) {
+    console.error(
+      "usage: node hash-composite-image.js (scripts | debs | s3 | registry)"
+    );
+    process.exit(1);
+  }
+  const mode = args[0];
+  console.log(await hashCompositeImage(mode));
   process.exit(0);
 }
 
-main().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});
+if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
+  main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}

tools/publish.bash

@@ -78,7 +78,7 @@ for lang in "${langs[@]}"; do
 done
 
 composite_local_hash="$(node tools/hash-composite-image.js scripts)"
-composite_remote_hash="$(node tools/hash-composite-image.js s3)"
+composite_remote_hash="$(node tools/hash-composite-image.js composite)"
 
 if [[ "${composite_local_hash}" != "${composite_remote_hash}" ]]; then
     make image push I=composite