diff --git a/Makefile b/Makefile index 541fc57..2770c03 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,10 @@ image: @: $${I} ifeq ($(I),composite) node tools/build-composite-image.js -else +else ifeq ($(I),admin) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) +else + docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --label riju.image-hash=$(shell node tools/hash-dockerfile.js $(I)) endif .PHONY: script @@ -39,8 +41,8 @@ script: node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash chmod +x $(BUILD)/build.bash -.PHONY: script-all -script-all: +.PHONY: scripts +scripts: node tools/make-foreach.js script .PHONY: pkg @@ -153,7 +155,7 @@ upload: @: $${L} $${T} $${S3_BUCKET} aws s3 rm --recursive $(S3_HASH) aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) - hash=$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash); test $${hash}; echo $${hash}; aws s3 cp - $(S3_HASH)/$${hash} < /dev/null + hash=aws s3 cp - $(S3_HASH)/$(shell dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash) < /dev/null .PHONY: publish publish: diff --git a/tools/docker-util.js b/tools/docker-util.js index dea5b44..ef310a9 100644 --- a/tools/docker-util.js +++ b/tools/docker-util.js @@ -37,8 +37,8 @@ export async function getLocalImageLabel(image, label) { return null; } } - const labels = JSON.stringify(output)[0].Config.Labels; - return labels[label] || null; + const labels = JSON.parse(output)[0].Config.Labels; + return (labels && labels[label]) || null; } // Return the value of a label on a Docker image that is on a remote @@ -47,11 +47,13 @@ export async function getRemoteImageLabel(image, label) { const [repo, tag] = image.split(":"); let output; try { - output = await runCommand(`skopeo inspect docker://${image}`, { - getStdout: true, - }); + output = ( + await runCommand(`skopeo inspect docker://${image}`, { + getStdout: true, + }) + ).stdout; } catch (err) { - const tags = JSON.stringify( + const tags = JSON.parse( ( await runCommand(`skopeo list-tags "docker://${repo}"`, { getStdout: true, @@ -69,7 +71,7 @@ export async function getRemoteImageLabel(image, label) { } } const labels = JSON.parse(output).Labels; - return labels[label] || null; + return (labels && labels[label]) || null; } // Return the value of $DOCKER_REPO, throwing an error if it's not set diff --git a/tools/hash-dockerfile.js b/tools/hash-dockerfile.js index 308a1da..a4c88ca 100644 --- a/tools/hash-dockerfile.js +++ b/tools/hash-dockerfile.js @@ -9,6 +9,7 @@ import dockerfileParser from "docker-file-parser"; import dockerignore from "@balena/dockerignore"; import _ from "lodash"; +import { getLocalImageLabel } from "./docker-util.js"; import { runCommand } from "./util.js"; // Given a string like "runtime" that identifies the relevant @@ -78,8 +79,11 @@ async function listFiles(path) { // opts is an optional config object. Keys: // * salt: additional arbitrary object which will be included verbatim // into the returned encoding object +// * hashLocalImages: truthy means that if a base image is not +// specified in dependentHashes then its hash will automatically +// be extracted from the labels of the local image by that name async function encodeDockerfile(name, dependentHashes, opts) { - const { salt } = opts || {}; + const { salt, hashLocalImages } = opts || {}; const dockerfile = await parseDockerfile(name); const ignore = await parseDockerignore(); const steps = await Promise.all( @@ -130,7 +134,11 @@ async function encodeDockerfile(name, dependentHashes, opts) { let image = args.split(" ")[0]; step.hash = dependentHashes[image]; if (!step.hash) { - throw new Error(`no hash given for base image: ${image}`); + if (hashLocalImages) { + step.hash = await getLocalImageLabel(image, "riju.image-hash"); + } else { + throw new Error(`no hash given for base image: ${image}`); + } } break; } @@ -153,6 +161,9 @@ async function encodeDockerfile(name, dependentHashes, opts) { // * salt: additional arbitrary object which will factor into the // generated hash, so the hash will change whenever the salt // changes +// * hashLocalImages: truthy means that if a base image is not +// specified in dependentHashes then its hash will automatically +// be extracted from the labels of the local image by that name export async function hashDockerfile(name, dependentHashes, opts) { const encoding = await encodeDockerfile(name, dependentHashes, opts); return crypto @@ -160,3 +171,25 @@ export async function hashDockerfile(name, dependentHashes, opts) { .update(JSON.stringify(encoding)) .digest("hex"); } + +// Parse command-line arguments, run main functionality, and exit. +async function main() { + const args = process.argv.slice(2); + if (args.length !== 1) { + console.error("usage: hash-dockerfile.js NAME"); + process.exit(1); + } + const [name] = args; + if (name === "composite") { + throw new Error("use build-composite-image.js instead for this"); + } + console.log(await hashDockerfile(name, {}, { hashLocalImages: true })); + process.exit(0); +} + +if (process.argv[1] === url.fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/tools/plan-publish.js b/tools/plan-publish.js index 6e6ea1f..a0a082e 100644 --- a/tools/plan-publish.js +++ b/tools/plan-publish.js @@ -23,7 +23,7 @@ async function planDockerImage(name, dependentHashes, opts) { `${DOCKER_REPO}:${name}`, "riju.image-hash" ); - dependentHashes[`${DOCKER_REPO}:${name}`] = desired; + dependentHashes[`riju:${name}`] = desired; return { artifact: "Docker image", name, @@ -73,7 +73,9 @@ async function planDebianPackages() { if (debExists) { local = ( - await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`) + await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { + getStdout: true, + }) ).stdout.trim() || null; } const remote = remoteHashes[name] || null; @@ -99,7 +101,9 @@ async function planDebianPackages() { } async function computePlan() { - const dependentHashes = {}; + const dependentHashes = { + "ubuntu:rolling": await getLocalImageDigest("ubuntu:rolling"), + }; const packaging = await planDockerImage("packaging", dependentHashes); const runtime = await planDockerImage("runtime", dependentHashes); const packages = await planDebianPackages(); @@ -134,7 +138,9 @@ async function main() { program.option("--all", "show also unchanged artifacts"); program.parse(process.argv); const plan = await computePlan(); - const filteredPlan = plan.filter(({ desired, remote }) => desired !== remote); + const filteredPlan = plan.filter( + ({ desired, local, remote }) => desired !== local || desired !== remote + ); console.log(); if (filteredPlan.length === 0) { console.log(`*** NO CHANGES REQUIRED TO ${plan.length} ARTIFACTS ***`); @@ -159,16 +165,16 @@ async function main() { func = () => {}; } else if (remote === desired && local !== desired) { action = "download remote"; - details = `${local} (local) => ${desired}`; + details = `${local} => ${desired}`; func = download; } else if (local === desired && remote !== desired) { action = "publish local"; - details = `${remote} (remote) => ${desired}`; + details = `${remote} => ${desired}`; func = upload; } else { action = "rebuild and publish"; if (local === remote) { - details = `${local} (local) => ${desired}`; + details = `${local} => ${desired}`; } else { details = `${local} (local), ${remote} (remote) => ${desired}`; } @@ -188,7 +194,7 @@ async function main() { ]); console.log(); if (program.publish) { - for ({ func } of tableData) { + for (const { func } of tableData) { await func(); } } diff --git a/tools/publish.bash b/tools/publish.bash index 9a9e7ca..9a28077 100755 --- a/tools/publish.bash +++ b/tools/publish.bash @@ -10,86 +10,9 @@ if [[ -z "${DEPLOY_SSH_PRIVATE_KEY:-}" ]]; then DEPLOY_SSH_PRIVATE_KEY="$(base64 < "${DEPLOY_SSH_PUBLIC_KEY_FILE%.pub}")" fi -make image push I=admin -make pull image push I=packaging +make pull-base scripts -declare -A published_hashes -while read line; do - pkg="$(awk -F/ '{ print $2 }' <<< "${line}")" - hash="$(awk -F/ '{ print $3 }' <<< "${line}")" - published_hashes["${pkg}"]="${hash}" -done < <(tools/list-s3-hashes.bash) - -readarray -t langs < <(ls langs | grep '\.yaml$' | grep -Eo '^[^.]+') - -for lang in "${langs[@]}"; do - for type in lang config; do - make script L="${lang}" T="${type}" - done -done - -fmt='%-31s %-8s %s\n' -printf "${fmt}" "PACKAGE" "CHANGE" "HASH" -printf "${fmt}" "-------" "------" "----" - -declare -A local_hashes -for lang in "${langs[@]}"; do - for type in lang config; do - pkg="riju-${type}-${lang}" - hash="$(sha1sum "build/${type}/${lang}/build.bash" | awk '{ print $1 }')" - local_hashes["${pkg}"]="${hash}" - published_hash="${published_hashes["${pkg}"]:-}" - if [[ -z "${published_hash}" ]]; then - printf "${fmt}" "${pkg}" "create" "${hash}" - elif [[ "${published_hash}" != "${hash}" ]]; then - printf "${fmt}" "${pkg}" "update" "${published_hash} => ${hash}" - else - printf "${fmt}" "${pkg}" "" "${hash}" - fi - done -done - -if [[ -n "${CONFIRM:-}" ]]; then - echo "Press enter to continue, or ctrl-C to abort..." - read -fi - -for lang in "${langs[@]}"; do - for type in lang config; do - pkg="riju-${type}-${lang}" - hash="${local_hashes["${pkg}"]}" - published_hash="${published_hashes["${pkg}"]:-}" - if [[ "${published_hash}" != "${hash}" ]]; then - make shell I=packaging CMD="make pkg L='${lang}' T='${type}'" - make upload L="${lang}" T="${type}" - fi - done -done - -for lang in "${langs[@]}"; do - for type in lang config; do - pkg="riju-${type}-${lang}" - hash="${local_hashes["${pkg}"]}" - published_hash="${published_hashes["${pkg}"]:-}" - if [[ "${published_hash}" == "${hash}" ]]; then - make download L="${lang}" T="${type}" - fi - done -done - -composite_scripts_hash="$(node tools/hash-composite-image.js scripts)" -composite_registry_hash="$(node tools/hash-composite-image.js registry)" - -if [[ "${composite_scripts_hash}" != "${composite_registry_hash}" ]]; then - make image push I=composite -else - make pull I=composite -fi - -make shell I=composite CMD="make test PATIENCE=4" - -make pull image push I=compile -make pull image push I=app +node tools/plan-publish.js --publish sha="$(git describe --match=always-omit-tag --always --abbrev=40 --dirty)"