From 5fc14b1403176eae216d62bd8ac0bc7b2bc74acf Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 27 Mar 2021 10:00:26 -0700 Subject: [PATCH] Compute all hashes correctly --- tools/depgraph.js | 78 +++++++++++-------- tools/docker-util.js | 38 +++++----- tools/plan-publish.js | 169 ------------------------------------------ 3 files changed, 65 insertions(+), 220 deletions(-) delete mode 100644 tools/plan-publish.js diff --git a/tools/depgraph.js b/tools/depgraph.js index a24f9fc..34bf579 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -17,6 +17,7 @@ import { getDockerRepo, getLocalImageLabel, getRemoteImageLabel, + getRemoteRepositoryTags, } from "./docker-util.js"; import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js"; import { runCommand } from "./util.js"; @@ -60,6 +61,9 @@ function getInformationalDependencies() { }) ); }, + dockerRepoTags: async () => { + return await getRemoteRepositoryTags(getDockerRepo()); + }, }; } @@ -88,13 +92,20 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { return { name: `image:${tag}`, dependencies: dependencies, + informationalDependencies: { + getPublishedHash: "dockerRepoTags", + }, getLocalHash: async () => { return await getLocalImageLabel(`riju:${tag}`, "riju.image-hash"); }, - getPublishedHash: async () => { + getPublishedHash: async ({ dockerRepoTags }) => { + if (!dockerRepoTags.includes(tag)) { + return null; + } return await getRemoteImageLabel( `${DOCKER_REPO}:${tag}`, - "riju.image-hash" + "riju.image-hash", + dockerRepoTags ); }, getDesiredHash: async (dependencyHashes) => { @@ -137,6 +148,7 @@ async function getDebArtifact({ type, lang }) { getPublishedHash: "s3DebHashes", }, getLocalHash: async () => { + const debPath = `build/${type}/${lang}/riju-${type}-${lang}.deb`; try { await fs.access(debPath); } catch (err) { @@ -228,7 +240,7 @@ async function getDeployArtifact(langs) { dependencies: ["image:app"] .concat(langs.map((lang) => `image:lang-${lang}`)) .concat(langs.map((lang) => `test:lang-${lang}`)), - publishOnly: true, + publishTarget: true, publishToRegistry: async () => { await runCommand(`tools/deploy.bash`); }, @@ -338,36 +350,42 @@ async function executeDepGraph({ desired: {}, }; for (const target of transitiveTargets) { - promises.local[target] = artifacts[target].getLocalHash(info); - promises.published[target] = artifacts[target].getPublishedHash(info); - promises.desired[target] = (async () => { - const dependencyHashes = {}; - for (const dependency of artifacts[target].dependencies) { - dependencyHashes[dependency] = await promises.desired[dependency]; - if (!dependencyHashes[dependency]) { - throw new Error( - `manual dependency must be built explicitly: dep ${target} --manual [--publish]` - ); + if (artifacts[target].publishTarget) { + promises.local[target] = Promise.resolve(null); + promises.published[target] = Promise.resolve(null); + promises.desired[target] = Promise.resolve(null); + } else { + promises.local[target] = artifacts[target].getLocalHash(info); + promises.published[target] = artifacts[target].getPublishedHash(info); + promises.desired[target] = (async () => { + const dependencyHashes = {}; + for (const dependency of artifacts[target].dependencies) { + dependencyHashes[dependency] = await promises.desired[dependency]; + if (!dependencyHashes[dependency]) { + throw new Error( + `manual dependency must be built explicitly: dep ${target} --manual [--publish]` + ); + } } - } - let hash = await artifacts[target].getDesiredHash(dependencyHashes); - if (hash || manual) { - return hash; - } - const promiseSets = [promises.published, promises.local]; - if (holdManual) { - promiseSets.reverse(); - } - for (const promiseSet of promiseSets) { - const hash = await promiseSet[target]; - if (hash) { + let hash = await artifacts[target].getDesiredHash(dependencyHashes); + if (hash || manual) { return hash; } - } - throw new Error( - `manual artifact must be built explicitly: dep ${target} --manual [--publish]` - ); - })(); + const promiseSets = [promises.published, promises.local]; + if (holdManual) { + promiseSets.reverse(); + } + for (const promiseSet of promiseSets) { + const hash = await promiseSet[target]; + if (hash) { + return hash; + } + } + throw new Error( + `manual artifact must be built explicitly: dep ${target} --manual [--publish]` + ); + })(); + } } await Promise.all( transitiveTargets.map(async (target) => { diff --git a/tools/docker-util.js b/tools/docker-util.js index ef310a9..7121d46 100644 --- a/tools/docker-util.js +++ b/tools/docker-util.js @@ -24,26 +24,29 @@ export async function getLocalImageLabel(image, label) { await runCommand(`docker inspect "${image}"`, { getStdout: true }) ).stdout; } catch (err) { - if ( - (await runCommand(`docker images -q "${image}"`, { getStdout: true })) - .stdout - ) { - // The image exists locally, something unexpected must have - // happened in docker inspect. - throw err; - } else { - // The image doesn't exist locally, that must be why docker - // inspect didn't work. - return null; - } + return null; } const labels = JSON.parse(output)[0].Config.Labels; return (labels && labels[label]) || null; } +// Return the list of tags in a remote Docker repository. +export async function getRemoteRepositoryTags(repo) { + return JSON.parse( + ( + await runCommand(`skopeo list-tags "docker://${repo}"`, { + getStdout: true, + }) + ).stdout + ).Tags; +} + // Return the value of a label on a Docker image that is on a remote -// registry. If the image or label doesn't exist, return null. -export async function getRemoteImageLabel(image, label) { +// registry. If the image or label doesn't exist, return null. You +// have to pass in a list of tags on the remote repository (see +// getRemoteRepositoryTags) so that we can distinguish between missing +// images and network errors. +export async function getRemoteImageLabel(image, label, tags) { const [repo, tag] = image.split(":"); let output; try { @@ -53,13 +56,6 @@ export async function getRemoteImageLabel(image, label) { }) ).stdout; } catch (err) { - const tags = JSON.parse( - ( - await runCommand(`skopeo list-tags "docker://${repo}"`, { - getStdout: true, - }) - ).stdout - ).Tags; if (tags.includes(tag)) { // Tag exists, something unexpected must have gone wrong when // running skopeo inspect. diff --git a/tools/plan-publish.js b/tools/plan-publish.js deleted file mode 100644 index 6ac8f68..0000000 --- a/tools/plan-publish.js +++ /dev/null @@ -1,169 +0,0 @@ -import crypto from "crypto"; -import { promises as fs } from "fs"; -import process from "process"; -import url from "url"; - -import { Command } from "commander"; -import _ from "lodash"; -import { v4 as getUUID } from "uuid"; - -import { getLangs, getPackages, readLangConfig } from "../lib/yaml.js"; -import { - getLocalImageDigest, - getLocalImageLabel, - getRemoteImageLabel, - getDockerRepo, -} from "./docker-util.js"; -import { hashDockerfile } from "./hash-dockerfile.js"; -import { runCommand } from "./util.js"; - -function printTable(data, headers) { - const widths = headers.map(({ key, title }) => - Math.max(title.length, ...data.map((datum) => datum[key].length)) - ); - [ - headers.map(({ title }) => title.toUpperCase()), - widths.map((width) => "-".repeat(width)), - ...data.map((datum) => headers.map(({ key }) => datum[key])), - ].map((values) => - console.log( - values.map((value, idx) => value.padEnd(widths[idx])).join(" ") - ) - ); -} - -// Parse command-line arguments, run main functionality, and exit. -async function main() { - const program = new Command(); - program.option("--execute", "pull and build artifacts locally"); - program.option( - "--publish", - "with --execute, also deploy newly built artifacts" - ); - program.option("--show-all", "show also unchanged artifacts"); - program.option( - "--omit-unneeded-downloads", - "don't download artifacts unless needed for dependent builds" - ); - program.parse(process.argv); - let plan = await computePlan(); - let tableData = plan.map( - ({ - id, - deps, - artifact, - name, - desired, - local, - remote, - download, - build, - upload, - }) => { - let action, details, func, couldPrune, noop; - if (remote === desired && local === desired) { - action = "(no action)"; - details = desired; - func = () => {}; - couldPrune = true; - noop = true; - } else if (remote === desired && local !== desired) { - action = "download remote"; - details = `${local} => ${desired}`; - func = download; - couldPrune = true; - noop = false; - } else if (local === desired && remote !== desired) { - action = "publish local"; - details = `${remote} => ${desired}`; - func = async () => { - if (program.publish) { - await upload(); - } - }; - couldPrune = false; - noop = false; - } else { - action = "rebuild and publish"; - if (local === remote) { - details = `${local} => ${desired}`; - } else { - details = `${local} (local), ${remote} (remote) => ${desired}`; - } - func = async () => { - await build(); - if (program.publish) { - await upload(); - } - }; - couldPrune = false; - noop = false; - } - return { - id, - deps, - couldPrune, - artifact, - name, - action, - details, - func, - noop, - }; - } - ); - if (program.omitUnneededDownloads) { - for (const datum of [...tableData].reverse()) { - if ( - datum.couldPrune && - _.every( - tableData, - (otherDatum) => - otherDatum.couldPrune || !otherDatum.deps.includes(datum.id) - ) - ) { - datum.pruned = true; - if (!datum.noop) { - datum.action += " [skipping]"; - datum.noop = true; - } - } - } - } - console.log(); - const filteredTableData = tableData.filter(({ noop }) => !noop); - if (filteredTableData.length === 0) { - console.log(`*** NO ACTION REQUIRED FOR ${plan.length} ARTIFACTS ***`); - } else { - console.log( - `*** ACTION REQUIRED FOR ${filteredTableData.length} of ${plan.length} ARTIFACTS ***` - ); - } - console.log(); - if (!program.showAll) { - tableData = filteredTableData; - } - if (tableData.length > 0) { - printTable(tableData, [ - { key: "artifact", title: "Type" }, - { key: "name", title: "Name" }, - { key: "action", title: "Action" }, - { key: "details", title: "Details" }, - ]); - console.log(); - } - tableData = filteredTableData; - if (program.execute) { - for (const { func } of tableData) { - await func(); - } - } - process.exit(0); -} - -if (process.argv[1] === url.fileURLToPath(import.meta.url)) { - main().catch((err) => { - console.error(err); - process.exit(1); - }); -}