Start getting the incremental deploy working

Makefile

@@ -28,8 +28,10 @@ image:
 	@: $${I}
 ifeq ($(I),composite)
 	node tools/build-composite-image.js
-else
+else ifeq ($(I),admin)
 	docker build . -f docker/$(I)/Dockerfile -t riju:$(I)
+else
+	docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --label riju.image-hash=$(shell node tools/hash-dockerfile.js $(I))
 endif
 
 .PHONY: script
@@ -39,8 +41,8 @@ script:
 	node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash
 	chmod +x $(BUILD)/build.bash
 
-.PHONY: script-all
-script-all:
+.PHONY: scripts
+scripts:
 	node tools/make-foreach.js script
 
 .PHONY: pkg
@@ -153,7 +155,7 @@ upload:
 	@: $${L} $${T} $${S3_BUCKET}
 	aws s3 rm --recursive $(S3_HASH)
 	aws s3 cp $(BUILD)/$(DEB) $(S3_DEB)
-	hash=$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash); test $${hash}; echo $${hash}; aws s3 cp - $(S3_HASH)/$${hash} < /dev/null
+	hash=aws s3 cp - $(S3_HASH)/$(shell dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash) < /dev/null
 
 .PHONY: publish
 publish:

tools/docker-util.js

@@ -37,8 +37,8 @@ export async function getLocalImageLabel(image, label) {
       return null;
     }
   }
-  const labels = JSON.stringify(output)[0].Config.Labels;
-  return labels[label] || null;
+  const labels = JSON.parse(output)[0].Config.Labels;
+  return (labels && labels[label]) || null;
 }
 
 // Return the value of a label on a Docker image that is on a remote
@@ -47,11 +47,13 @@ export async function getRemoteImageLabel(image, label) {
   const [repo, tag] = image.split(":");
   let output;
   try {
-    output = await runCommand(`skopeo inspect docker://${image}`, {
-      getStdout: true,
-    });
+    output = (
+      await runCommand(`skopeo inspect docker://${image}`, {
+        getStdout: true,
+      })
+    ).stdout;
   } catch (err) {
-    const tags = JSON.stringify(
+    const tags = JSON.parse(
       (
         await runCommand(`skopeo list-tags "docker://${repo}"`, {
           getStdout: true,
@@ -69,7 +71,7 @@ export async function getRemoteImageLabel(image, label) {
     }
   }
   const labels = JSON.parse(output).Labels;
-  return labels[label] || null;
+  return (labels && labels[label]) || null;
 }
 
 // Return the value of $DOCKER_REPO, throwing an error if it's not set

tools/hash-dockerfile.js

@@ -9,6 +9,7 @@ import dockerfileParser from "docker-file-parser";
 import dockerignore from "@balena/dockerignore";
 import _ from "lodash";
 
+import { getLocalImageLabel } from "./docker-util.js";
 import { runCommand } from "./util.js";
 
 // Given a string like "runtime" that identifies the relevant
@@ -78,8 +79,11 @@ async function listFiles(path) {
 // opts is an optional config object. Keys:
 // * salt: additional arbitrary object which will be included verbatim
 //     into the returned encoding object
+// * hashLocalImages: truthy means that if a base image is not
+//     specified in dependentHashes then its hash will automatically
+//     be extracted from the labels of the local image by that name
 async function encodeDockerfile(name, dependentHashes, opts) {
-  const { salt } = opts || {};
+  const { salt, hashLocalImages } = opts || {};
   const dockerfile = await parseDockerfile(name);
   const ignore = await parseDockerignore();
   const steps = await Promise.all(
@@ -130,7 +134,11 @@ async function encodeDockerfile(name, dependentHashes, opts) {
           let image = args.split(" ")[0];
           step.hash = dependentHashes[image];
           if (!step.hash) {
-            throw new Error(`no hash given for base image: ${image}`);
+            if (hashLocalImages) {
+              step.hash = await getLocalImageLabel(image, "riju.image-hash");
+            } else {
+              throw new Error(`no hash given for base image: ${image}`);
+            }
           }
           break;
       }
@@ -153,6 +161,9 @@ async function encodeDockerfile(name, dependentHashes, opts) {
 // * salt: additional arbitrary object which will factor into the
 //     generated hash, so the hash will change whenever the salt
 //     changes
+// * hashLocalImages: truthy means that if a base image is not
+//     specified in dependentHashes then its hash will automatically
+//     be extracted from the labels of the local image by that name
 export async function hashDockerfile(name, dependentHashes, opts) {
   const encoding = await encodeDockerfile(name, dependentHashes, opts);
   return crypto
@@ -160,3 +171,25 @@ export async function hashDockerfile(name, dependentHashes, opts) {
     .update(JSON.stringify(encoding))
     .digest("hex");
 }
+
+// Parse command-line arguments, run main functionality, and exit.
+async function main() {
+  const args = process.argv.slice(2);
+  if (args.length !== 1) {
+    console.error("usage: hash-dockerfile.js NAME");
+    process.exit(1);
+  }
+  const [name] = args;
+  if (name === "composite") {
+    throw new Error("use build-composite-image.js instead for this");
+  }
+  console.log(await hashDockerfile(name, {}, { hashLocalImages: true }));
+  process.exit(0);
+}
+
+if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
+  main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}

tools/plan-publish.js

@@ -23,7 +23,7 @@ async function planDockerImage(name, dependentHashes, opts) {
     `${DOCKER_REPO}:${name}`,
     "riju.image-hash"
   );
-  dependentHashes[`${DOCKER_REPO}:${name}`] = desired;
+  dependentHashes[`riju:${name}`] = desired;
   return {
     artifact: "Docker image",
     name,
@@ -73,7 +73,9 @@ async function planDebianPackages() {
         if (debExists) {
           local =
             (
-              await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`)
+              await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
+                getStdout: true,
+              })
             ).stdout.trim() || null;
         }
         const remote = remoteHashes[name] || null;
@@ -99,7 +101,9 @@ async function planDebianPackages() {
 }
 
 async function computePlan() {
-  const dependentHashes = {};
+  const dependentHashes = {
+    "ubuntu:rolling": await getLocalImageDigest("ubuntu:rolling"),
+  };
   const packaging = await planDockerImage("packaging", dependentHashes);
   const runtime = await planDockerImage("runtime", dependentHashes);
   const packages = await planDebianPackages();
@@ -134,7 +138,9 @@ async function main() {
   program.option("--all", "show also unchanged artifacts");
   program.parse(process.argv);
   const plan = await computePlan();
-  const filteredPlan = plan.filter(({ desired, remote }) => desired !== remote);
+  const filteredPlan = plan.filter(
+    ({ desired, local, remote }) => desired !== local || desired !== remote
+  );
   console.log();
   if (filteredPlan.length === 0) {
     console.log(`*** NO CHANGES REQUIRED TO ${plan.length} ARTIFACTS ***`);
@@ -159,16 +165,16 @@ async function main() {
         func = () => {};
       } else if (remote === desired && local !== desired) {
         action = "download remote";
-        details = `${local} (local) => ${desired}`;
+        details = `${local} => ${desired}`;
         func = download;
       } else if (local === desired && remote !== desired) {
         action = "publish local";
-        details = `${remote} (remote) => ${desired}`;
+        details = `${remote} => ${desired}`;
         func = upload;
       } else {
         action = "rebuild and publish";
         if (local === remote) {
-          details = `${local} (local) => ${desired}`;
+          details = `${local} => ${desired}`;
         } else {
           details = `${local} (local), ${remote} (remote) => ${desired}`;
         }
@@ -188,7 +194,7 @@ async function main() {
   ]);
   console.log();
   if (program.publish) {
-    for ({ func } of tableData) {
+    for (const { func } of tableData) {
       await func();
     }
   }

tools/publish.bash

@@ -10,86 +10,9 @@ if [[ -z "${DEPLOY_SSH_PRIVATE_KEY:-}" ]]; then
     DEPLOY_SSH_PRIVATE_KEY="$(base64 < "${DEPLOY_SSH_PUBLIC_KEY_FILE%.pub}")"
 fi
 
-make image push I=admin
-make pull image push I=packaging
+make pull-base scripts
 
-declare -A published_hashes
-while read line; do
-    pkg="$(awk -F/ '{ print $2 }' <<< "${line}")"
-    hash="$(awk -F/ '{ print $3 }' <<< "${line}")"
-    published_hashes["${pkg}"]="${hash}"
-done < <(tools/list-s3-hashes.bash)
-
-readarray -t langs < <(ls langs | grep '\.yaml$' | grep -Eo '^[^.]+')
-
-for lang in "${langs[@]}"; do
-    for type in lang config; do
-        make script L="${lang}" T="${type}"
-    done
-done
-
-fmt='%-31s %-8s %s\n'
-printf "${fmt}" "PACKAGE" "CHANGE" "HASH"
-printf "${fmt}" "-------" "------" "----"
-
-declare -A local_hashes
-for lang in "${langs[@]}"; do
-    for type in lang config; do
-        pkg="riju-${type}-${lang}"
-        hash="$(sha1sum "build/${type}/${lang}/build.bash" | awk '{ print $1 }')"
-        local_hashes["${pkg}"]="${hash}"
-        published_hash="${published_hashes["${pkg}"]:-}"
-        if [[ -z "${published_hash}" ]]; then
-            printf "${fmt}" "${pkg}" "create" "${hash}"
-        elif [[ "${published_hash}" != "${hash}" ]]; then
-            printf "${fmt}" "${pkg}" "update" "${published_hash} => ${hash}"
-        else
-            printf "${fmt}" "${pkg}" "" "${hash}"
-        fi
-    done
-done
-
-if [[ -n "${CONFIRM:-}" ]]; then
-    echo "Press enter to continue, or ctrl-C to abort..."
-    read
-fi
-
-for lang in "${langs[@]}"; do
-    for type in lang config; do
-        pkg="riju-${type}-${lang}"
-        hash="${local_hashes["${pkg}"]}"
-        published_hash="${published_hashes["${pkg}"]:-}"
-        if [[ "${published_hash}" != "${hash}" ]]; then
-            make shell I=packaging CMD="make pkg L='${lang}' T='${type}'"
-            make upload L="${lang}" T="${type}"
-        fi
-    done
-done
-
-for lang in "${langs[@]}"; do
-    for type in lang config; do
-        pkg="riju-${type}-${lang}"
-        hash="${local_hashes["${pkg}"]}"
-        published_hash="${published_hashes["${pkg}"]:-}"
-        if [[ "${published_hash}" == "${hash}" ]]; then
-            make download L="${lang}" T="${type}"
-        fi
-    done
-done
-
-composite_scripts_hash="$(node tools/hash-composite-image.js scripts)"
-composite_registry_hash="$(node tools/hash-composite-image.js registry)"
-
-if [[ "${composite_scripts_hash}" != "${composite_registry_hash}" ]]; then
-    make image push I=composite
-else
-    make pull I=composite
-fi
-
-make shell I=composite CMD="make test PATIENCE=4"
-
-make pull image push I=compile
-make pull image push I=app
+node tools/plan-publish.js --publish
 
 sha="$(git describe --match=always-omit-tag --always --abbrev=40 --dirty)"