Compute all hashes correctly
This commit is contained in:
Radon Rosborough
parent
5f523f8142
commit
5fc14b1403
|
|
@ -17,6 +17,7 @@ import {
|
|||
getDockerRepo,
|
||||
getLocalImageLabel,
|
||||
getRemoteImageLabel,
|
||||
getRemoteRepositoryTags,
|
||||
} from "./docker-util.js";
|
||||
import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js";
|
||||
import { runCommand } from "./util.js";
|
||||
|
|
@ -60,6 +61,9 @@ function getInformationalDependencies() {
|
|||
})
|
||||
);
|
||||
},
|
||||
dockerRepoTags: async () => {
|
||||
return await getRemoteRepositoryTags(getDockerRepo());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -88,13 +92,20 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) {
|
|||
return {
|
||||
name: `image:${tag}`,
|
||||
dependencies: dependencies,
|
||||
informationalDependencies: {
|
||||
getPublishedHash: "dockerRepoTags",
|
||||
},
|
||||
getLocalHash: async () => {
|
||||
return await getLocalImageLabel(`riju:${tag}`, "riju.image-hash");
|
||||
},
|
||||
getPublishedHash: async () => {
|
||||
getPublishedHash: async ({ dockerRepoTags }) => {
|
||||
if (!dockerRepoTags.includes(tag)) {
|
||||
return null;
|
||||
}
|
||||
return await getRemoteImageLabel(
|
||||
`${DOCKER_REPO}:${tag}`,
|
||||
"riju.image-hash"
|
||||
"riju.image-hash",
|
||||
dockerRepoTags
|
||||
);
|
||||
},
|
||||
getDesiredHash: async (dependencyHashes) => {
|
||||
|
|
@ -137,6 +148,7 @@ async function getDebArtifact({ type, lang }) {
|
|||
getPublishedHash: "s3DebHashes",
|
||||
},
|
||||
getLocalHash: async () => {
|
||||
const debPath = `build/${type}/${lang}/riju-${type}-${lang}.deb`;
|
||||
try {
|
||||
await fs.access(debPath);
|
||||
} catch (err) {
|
||||
|
|
@ -228,7 +240,7 @@ async function getDeployArtifact(langs) {
|
|||
dependencies: ["image:app"]
|
||||
.concat(langs.map((lang) => `image:lang-${lang}`))
|
||||
.concat(langs.map((lang) => `test:lang-${lang}`)),
|
||||
publishOnly: true,
|
||||
publishTarget: true,
|
||||
publishToRegistry: async () => {
|
||||
await runCommand(`tools/deploy.bash`);
|
||||
},
|
||||
|
|
@ -338,36 +350,42 @@ async function executeDepGraph({
|
|||
desired: {},
|
||||
};
|
||||
for (const target of transitiveTargets) {
|
||||
promises.local[target] = artifacts[target].getLocalHash(info);
|
||||
promises.published[target] = artifacts[target].getPublishedHash(info);
|
||||
promises.desired[target] = (async () => {
|
||||
const dependencyHashes = {};
|
||||
for (const dependency of artifacts[target].dependencies) {
|
||||
dependencyHashes[dependency] = await promises.desired[dependency];
|
||||
if (!dependencyHashes[dependency]) {
|
||||
throw new Error(
|
||||
`manual dependency must be built explicitly: dep ${target} --manual [--publish]`
|
||||
);
|
||||
if (artifacts[target].publishTarget) {
|
||||
promises.local[target] = Promise.resolve(null);
|
||||
promises.published[target] = Promise.resolve(null);
|
||||
promises.desired[target] = Promise.resolve(null);
|
||||
} else {
|
||||
promises.local[target] = artifacts[target].getLocalHash(info);
|
||||
promises.published[target] = artifacts[target].getPublishedHash(info);
|
||||
promises.desired[target] = (async () => {
|
||||
const dependencyHashes = {};
|
||||
for (const dependency of artifacts[target].dependencies) {
|
||||
dependencyHashes[dependency] = await promises.desired[dependency];
|
||||
if (!dependencyHashes[dependency]) {
|
||||
throw new Error(
|
||||
`manual dependency must be built explicitly: dep ${target} --manual [--publish]`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let hash = await artifacts[target].getDesiredHash(dependencyHashes);
|
||||
if (hash || manual) {
|
||||
return hash;
|
||||
}
|
||||
const promiseSets = [promises.published, promises.local];
|
||||
if (holdManual) {
|
||||
promiseSets.reverse();
|
||||
}
|
||||
for (const promiseSet of promiseSets) {
|
||||
const hash = await promiseSet[target];
|
||||
if (hash) {
|
||||
let hash = await artifacts[target].getDesiredHash(dependencyHashes);
|
||||
if (hash || manual) {
|
||||
return hash;
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`manual artifact must be built explicitly: dep ${target} --manual [--publish]`
|
||||
);
|
||||
})();
|
||||
const promiseSets = [promises.published, promises.local];
|
||||
if (holdManual) {
|
||||
promiseSets.reverse();
|
||||
}
|
||||
for (const promiseSet of promiseSets) {
|
||||
const hash = await promiseSet[target];
|
||||
if (hash) {
|
||||
return hash;
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`manual artifact must be built explicitly: dep ${target} --manual [--publish]`
|
||||
);
|
||||
})();
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
transitiveTargets.map(async (target) => {
|
||||
|
|
|
|||
|
|
@ -24,26 +24,29 @@ export async function getLocalImageLabel(image, label) {
|
|||
await runCommand(`docker inspect "${image}"`, { getStdout: true })
|
||||
).stdout;
|
||||
} catch (err) {
|
||||
if (
|
||||
(await runCommand(`docker images -q "${image}"`, { getStdout: true }))
|
||||
.stdout
|
||||
) {
|
||||
// The image exists locally, something unexpected must have
|
||||
// happened in docker inspect.
|
||||
throw err;
|
||||
} else {
|
||||
// The image doesn't exist locally, that must be why docker
|
||||
// inspect didn't work.
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const labels = JSON.parse(output)[0].Config.Labels;
|
||||
return (labels && labels[label]) || null;
|
||||
}
|
||||
|
||||
// Return the list of tags in a remote Docker repository.
|
||||
export async function getRemoteRepositoryTags(repo) {
|
||||
return JSON.parse(
|
||||
(
|
||||
await runCommand(`skopeo list-tags "docker://${repo}"`, {
|
||||
getStdout: true,
|
||||
})
|
||||
).stdout
|
||||
).Tags;
|
||||
}
|
||||
|
||||
// Return the value of a label on a Docker image that is on a remote
|
||||
// registry. If the image or label doesn't exist, return null.
|
||||
export async function getRemoteImageLabel(image, label) {
|
||||
// registry. If the image or label doesn't exist, return null. You
|
||||
// have to pass in a list of tags on the remote repository (see
|
||||
// getRemoteRepositoryTags) so that we can distinguish between missing
|
||||
// images and network errors.
|
||||
export async function getRemoteImageLabel(image, label, tags) {
|
||||
const [repo, tag] = image.split(":");
|
||||
let output;
|
||||
try {
|
||||
|
|
@ -53,13 +56,6 @@ export async function getRemoteImageLabel(image, label) {
|
|||
})
|
||||
).stdout;
|
||||
} catch (err) {
|
||||
const tags = JSON.parse(
|
||||
(
|
||||
await runCommand(`skopeo list-tags "docker://${repo}"`, {
|
||||
getStdout: true,
|
||||
})
|
||||
).stdout
|
||||
).Tags;
|
||||
if (tags.includes(tag)) {
|
||||
// Tag exists, something unexpected must have gone wrong when
|
||||
// running skopeo inspect.
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
import crypto from "crypto";
|
||||
import { promises as fs } from "fs";
|
||||
import process from "process";
|
||||
import url from "url";
|
||||
|
||||
import { Command } from "commander";
|
||||
import _ from "lodash";
|
||||
import { v4 as getUUID } from "uuid";
|
||||
|
||||
import { getLangs, getPackages, readLangConfig } from "../lib/yaml.js";
|
||||
import {
|
||||
getLocalImageDigest,
|
||||
getLocalImageLabel,
|
||||
getRemoteImageLabel,
|
||||
getDockerRepo,
|
||||
} from "./docker-util.js";
|
||||
import { hashDockerfile } from "./hash-dockerfile.js";
|
||||
import { runCommand } from "./util.js";
|
||||
|
||||
function printTable(data, headers) {
|
||||
const widths = headers.map(({ key, title }) =>
|
||||
Math.max(title.length, ...data.map((datum) => datum[key].length))
|
||||
);
|
||||
[
|
||||
headers.map(({ title }) => title.toUpperCase()),
|
||||
widths.map((width) => "-".repeat(width)),
|
||||
...data.map((datum) => headers.map(({ key }) => datum[key])),
|
||||
].map((values) =>
|
||||
console.log(
|
||||
values.map((value, idx) => value.padEnd(widths[idx])).join(" ")
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Parse command-line arguments, run main functionality, and exit.
|
||||
async function main() {
|
||||
const program = new Command();
|
||||
program.option("--execute", "pull and build artifacts locally");
|
||||
program.option(
|
||||
"--publish",
|
||||
"with --execute, also deploy newly built artifacts"
|
||||
);
|
||||
program.option("--show-all", "show also unchanged artifacts");
|
||||
program.option(
|
||||
"--omit-unneeded-downloads",
|
||||
"don't download artifacts unless needed for dependent builds"
|
||||
);
|
||||
program.parse(process.argv);
|
||||
let plan = await computePlan();
|
||||
let tableData = plan.map(
|
||||
({
|
||||
id,
|
||||
deps,
|
||||
artifact,
|
||||
name,
|
||||
desired,
|
||||
local,
|
||||
remote,
|
||||
download,
|
||||
build,
|
||||
upload,
|
||||
}) => {
|
||||
let action, details, func, couldPrune, noop;
|
||||
if (remote === desired && local === desired) {
|
||||
action = "(no action)";
|
||||
details = desired;
|
||||
func = () => {};
|
||||
couldPrune = true;
|
||||
noop = true;
|
||||
} else if (remote === desired && local !== desired) {
|
||||
action = "download remote";
|
||||
details = `${local} => ${desired}`;
|
||||
func = download;
|
||||
couldPrune = true;
|
||||
noop = false;
|
||||
} else if (local === desired && remote !== desired) {
|
||||
action = "publish local";
|
||||
details = `${remote} => ${desired}`;
|
||||
func = async () => {
|
||||
if (program.publish) {
|
||||
await upload();
|
||||
}
|
||||
};
|
||||
couldPrune = false;
|
||||
noop = false;
|
||||
} else {
|
||||
action = "rebuild and publish";
|
||||
if (local === remote) {
|
||||
details = `${local} => ${desired}`;
|
||||
} else {
|
||||
details = `${local} (local), ${remote} (remote) => ${desired}`;
|
||||
}
|
||||
func = async () => {
|
||||
await build();
|
||||
if (program.publish) {
|
||||
await upload();
|
||||
}
|
||||
};
|
||||
couldPrune = false;
|
||||
noop = false;
|
||||
}
|
||||
return {
|
||||
id,
|
||||
deps,
|
||||
couldPrune,
|
||||
artifact,
|
||||
name,
|
||||
action,
|
||||
details,
|
||||
func,
|
||||
noop,
|
||||
};
|
||||
}
|
||||
);
|
||||
if (program.omitUnneededDownloads) {
|
||||
for (const datum of [...tableData].reverse()) {
|
||||
if (
|
||||
datum.couldPrune &&
|
||||
_.every(
|
||||
tableData,
|
||||
(otherDatum) =>
|
||||
otherDatum.couldPrune || !otherDatum.deps.includes(datum.id)
|
||||
)
|
||||
) {
|
||||
datum.pruned = true;
|
||||
if (!datum.noop) {
|
||||
datum.action += " [skipping]";
|
||||
datum.noop = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log();
|
||||
const filteredTableData = tableData.filter(({ noop }) => !noop);
|
||||
if (filteredTableData.length === 0) {
|
||||
console.log(`*** NO ACTION REQUIRED FOR ${plan.length} ARTIFACTS ***`);
|
||||
} else {
|
||||
console.log(
|
||||
`*** ACTION REQUIRED FOR ${filteredTableData.length} of ${plan.length} ARTIFACTS ***`
|
||||
);
|
||||
}
|
||||
console.log();
|
||||
if (!program.showAll) {
|
||||
tableData = filteredTableData;
|
||||
}
|
||||
if (tableData.length > 0) {
|
||||
printTable(tableData, [
|
||||
{ key: "artifact", title: "Type" },
|
||||
{ key: "name", title: "Name" },
|
||||
{ key: "action", title: "Action" },
|
||||
{ key: "details", title: "Details" },
|
||||
]);
|
||||
console.log();
|
||||
}
|
||||
tableData = filteredTableData;
|
||||
if (program.execute) {
|
||||
for (const { func } of tableData) {
|
||||
await func();
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Loading…
Reference in New Issue