Compute all hashes correctly

tools/depgraph.js

@@ -17,6 +17,7 @@ import {
   getDockerRepo,
   getLocalImageLabel,
   getRemoteImageLabel,
+  getRemoteRepositoryTags,
 } from "./docker-util.js";
 import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js";
 import { runCommand } from "./util.js";
@@ -60,6 +61,9 @@ function getInformationalDependencies() {
         })
       );
     },
+    dockerRepoTags: async () => {
+      return await getRemoteRepositoryTags(getDockerRepo());
+    },
   };
 }
 
@@ -88,13 +92,20 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) {
   return {
     name: `image:${tag}`,
     dependencies: dependencies,
+    informationalDependencies: {
+      getPublishedHash: "dockerRepoTags",
+    },
     getLocalHash: async () => {
       return await getLocalImageLabel(`riju:${tag}`, "riju.image-hash");
     },
-    getPublishedHash: async () => {
+    getPublishedHash: async ({ dockerRepoTags }) => {
+      if (!dockerRepoTags.includes(tag)) {
+        return null;
+      }
       return await getRemoteImageLabel(
         `${DOCKER_REPO}:${tag}`,
-        "riju.image-hash"
+        "riju.image-hash",
+        dockerRepoTags
       );
     },
     getDesiredHash: async (dependencyHashes) => {
@@ -137,6 +148,7 @@ async function getDebArtifact({ type, lang }) {
       getPublishedHash: "s3DebHashes",
     },
     getLocalHash: async () => {
+      const debPath = `build/${type}/${lang}/riju-${type}-${lang}.deb`;
       try {
         await fs.access(debPath);
       } catch (err) {
@@ -228,7 +240,7 @@ async function getDeployArtifact(langs) {
     dependencies: ["image:app"]
       .concat(langs.map((lang) => `image:lang-${lang}`))
       .concat(langs.map((lang) => `test:lang-${lang}`)),
-    publishOnly: true,
+    publishTarget: true,
     publishToRegistry: async () => {
       await runCommand(`tools/deploy.bash`);
     },
@@ -338,36 +350,42 @@ async function executeDepGraph({
     desired: {},
   };
   for (const target of transitiveTargets) {
-    promises.local[target] = artifacts[target].getLocalHash(info);
+    if (artifacts[target].publishTarget) {
-    promises.published[target] = artifacts[target].getPublishedHash(info);
+      promises.local[target] = Promise.resolve(null);
-    promises.desired[target] = (async () => {
+      promises.published[target] = Promise.resolve(null);
-      const dependencyHashes = {};
+      promises.desired[target] = Promise.resolve(null);
-      for (const dependency of artifacts[target].dependencies) {
+    } else {
-        dependencyHashes[dependency] = await promises.desired[dependency];
+      promises.local[target] = artifacts[target].getLocalHash(info);
-        if (!dependencyHashes[dependency]) {
+      promises.published[target] = artifacts[target].getPublishedHash(info);
-          throw new Error(
+      promises.desired[target] = (async () => {
-            `manual dependency must be built explicitly: dep ${target} --manual [--publish]`
+        const dependencyHashes = {};
-          );
+        for (const dependency of artifacts[target].dependencies) {
+          dependencyHashes[dependency] = await promises.desired[dependency];
+          if (!dependencyHashes[dependency]) {
+            throw new Error(
+              `manual dependency must be built explicitly: dep ${target} --manual [--publish]`
+            );
+          }
         }
-      }
+        let hash = await artifacts[target].getDesiredHash(dependencyHashes);
-      let hash = await artifacts[target].getDesiredHash(dependencyHashes);
+        if (hash || manual) {
-      if (hash || manual) {
-        return hash;
-      }
-      const promiseSets = [promises.published, promises.local];
-      if (holdManual) {
-        promiseSets.reverse();
-      }
-      for (const promiseSet of promiseSets) {
-        const hash = await promiseSet[target];
-        if (hash) {
           return hash;
         }
-      }
+        const promiseSets = [promises.published, promises.local];
-      throw new Error(
+        if (holdManual) {
-        `manual artifact must be built explicitly: dep ${target} --manual [--publish]`
+          promiseSets.reverse();
-      );
+        }
-    })();
+        for (const promiseSet of promiseSets) {
+          const hash = await promiseSet[target];
+          if (hash) {
+            return hash;
+          }
+        }
+        throw new Error(
+          `manual artifact must be built explicitly: dep ${target} --manual [--publish]`
+        );
+      })();
+    }
   }
   await Promise.all(
     transitiveTargets.map(async (target) => {

tools/docker-util.js

@@ -24,26 +24,29 @@ export async function getLocalImageLabel(image, label) {
       await runCommand(`docker inspect "${image}"`, { getStdout: true })
     ).stdout;
   } catch (err) {
-    if (
+    return null;
-      (await runCommand(`docker images -q "${image}"`, { getStdout: true }))
-        .stdout
-    ) {
-      // The image exists locally, something unexpected must have
-      // happened in docker inspect.
-      throw err;
-    } else {
-      // The image doesn't exist locally, that must be why docker
-      // inspect didn't work.
-      return null;
-    }
   }
   const labels = JSON.parse(output)[0].Config.Labels;
   return (labels && labels[label]) || null;
 }
 
+// Return the list of tags in a remote Docker repository.
+export async function getRemoteRepositoryTags(repo) {
+  return JSON.parse(
+    (
+      await runCommand(`skopeo list-tags "docker://${repo}"`, {
+        getStdout: true,
+      })
+    ).stdout
+  ).Tags;
+}
+
 // Return the value of a label on a Docker image that is on a remote
-// registry. If the image or label doesn't exist, return null.
+// registry. If the image or label doesn't exist, return null. You
-export async function getRemoteImageLabel(image, label) {
+// have to pass in a list of tags on the remote repository (see
+// getRemoteRepositoryTags) so that we can distinguish between missing
+// images and network errors.
+export async function getRemoteImageLabel(image, label, tags) {
   const [repo, tag] = image.split(":");
   let output;
   try {
@@ -53,13 +56,6 @@ export async function getRemoteImageLabel(image, label) {
       })
     ).stdout;
   } catch (err) {
-    const tags = JSON.parse(
-      (
-        await runCommand(`skopeo list-tags "docker://${repo}"`, {
-          getStdout: true,
-        })
-      ).stdout
-    ).Tags;
     if (tags.includes(tag)) {
       // Tag exists, something unexpected must have gone wrong when
       // running skopeo inspect.

tools/plan-publish.js

@@ -1,169 +0,0 @@
-import crypto from "crypto";
-import { promises as fs } from "fs";
-import process from "process";
-import url from "url";
-
-import { Command } from "commander";
-import _ from "lodash";
-import { v4 as getUUID } from "uuid";
-
-import { getLangs, getPackages, readLangConfig } from "../lib/yaml.js";
-import {
-  getLocalImageDigest,
-  getLocalImageLabel,
-  getRemoteImageLabel,
-  getDockerRepo,
-} from "./docker-util.js";
-import { hashDockerfile } from "./hash-dockerfile.js";
-import { runCommand } from "./util.js";
-
-function printTable(data, headers) {
-  const widths = headers.map(({ key, title }) =>
-    Math.max(title.length, ...data.map((datum) => datum[key].length))
-  );
-  [
-    headers.map(({ title }) => title.toUpperCase()),
-    widths.map((width) => "-".repeat(width)),
-    ...data.map((datum) => headers.map(({ key }) => datum[key])),
-  ].map((values) =>
-    console.log(
-      values.map((value, idx) => value.padEnd(widths[idx])).join("  ")
-    )
-  );
-}
-
-// Parse command-line arguments, run main functionality, and exit.
-async function main() {
-  const program = new Command();
-  program.option("--execute", "pull and build artifacts locally");
-  program.option(
-    "--publish",
-    "with --execute, also deploy newly built artifacts"
-  );
-  program.option("--show-all", "show also unchanged artifacts");
-  program.option(
-    "--omit-unneeded-downloads",
-    "don't download artifacts unless needed for dependent builds"
-  );
-  program.parse(process.argv);
-  let plan = await computePlan();
-  let tableData = plan.map(
-    ({
-      id,
-      deps,
-      artifact,
-      name,
-      desired,
-      local,
-      remote,
-      download,
-      build,
-      upload,
-    }) => {
-      let action, details, func, couldPrune, noop;
-      if (remote === desired && local === desired) {
-        action = "(no action)";
-        details = desired;
-        func = () => {};
-        couldPrune = true;
-        noop = true;
-      } else if (remote === desired && local !== desired) {
-        action = "download remote";
-        details = `${local} => ${desired}`;
-        func = download;
-        couldPrune = true;
-        noop = false;
-      } else if (local === desired && remote !== desired) {
-        action = "publish local";
-        details = `${remote} => ${desired}`;
-        func = async () => {
-          if (program.publish) {
-            await upload();
-          }
-        };
-        couldPrune = false;
-        noop = false;
-      } else {
-        action = "rebuild and publish";
-        if (local === remote) {
-          details = `${local} => ${desired}`;
-        } else {
-          details = `${local} (local), ${remote} (remote) => ${desired}`;
-        }
-        func = async () => {
-          await build();
-          if (program.publish) {
-            await upload();
-          }
-        };
-        couldPrune = false;
-        noop = false;
-      }
-      return {
-        id,
-        deps,
-        couldPrune,
-        artifact,
-        name,
-        action,
-        details,
-        func,
-        noop,
-      };
-    }
-  );
-  if (program.omitUnneededDownloads) {
-    for (const datum of [...tableData].reverse()) {
-      if (
-        datum.couldPrune &&
-        _.every(
-          tableData,
-          (otherDatum) =>
-            otherDatum.couldPrune || !otherDatum.deps.includes(datum.id)
-        )
-      ) {
-        datum.pruned = true;
-        if (!datum.noop) {
-          datum.action += " [skipping]";
-          datum.noop = true;
-        }
-      }
-    }
-  }
-  console.log();
-  const filteredTableData = tableData.filter(({ noop }) => !noop);
-  if (filteredTableData.length === 0) {
-    console.log(`*** NO ACTION REQUIRED FOR ${plan.length} ARTIFACTS ***`);
-  } else {
-    console.log(
-      `*** ACTION REQUIRED FOR ${filteredTableData.length} of ${plan.length} ARTIFACTS ***`
-    );
-  }
-  console.log();
-  if (!program.showAll) {
-    tableData = filteredTableData;
-  }
-  if (tableData.length > 0) {
-    printTable(tableData, [
-      { key: "artifact", title: "Type" },
-      { key: "name", title: "Name" },
-      { key: "action", title: "Action" },
-      { key: "details", title: "Details" },
-    ]);
-    console.log();
-  }
-  tableData = filteredTableData;
-  if (program.execute) {
-    for (const { func } of tableData) {
-      await func();
-    }
-  }
-  process.exit(0);
-}
-
-if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
-  main().catch((err) => {
-    console.error(err);
-    process.exit(1);
-  });
-}