From 77387027f2abb99ceaa9a0b64e03c62096b3ea56 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 26 Dec 2020 19:49:01 -0800 Subject: [PATCH] Set up initial CI configuration --- .circleci/config.yml | 17 +++++++ Makefile | 33 ++++++------ docker/admin/install.bash | 1 + tf/infra.tf | 38 ++++++++++++++ tools/build-composite-image.js | 3 ++ tools/ci-bootstrap.bash | 14 ++++++ tools/ci-run.bash | 8 +++ tools/hash-composite-image.js | 64 ++++++++++++++++++++++++ tools/list-s3-hashes.bash | 5 ++ tools/publish.bash | 91 ++++++++++++++++++++++++++++++++++ tools/util.js | 44 ++++++++++++---- 11 files changed, 293 insertions(+), 25 deletions(-) create mode 100644 .circleci/config.yml create mode 100755 tools/ci-bootstrap.bash create mode 100755 tools/ci-run.bash create mode 100644 tools/hash-composite-image.js create mode 100755 tools/list-s3-hashes.bash create mode 100755 tools/publish.bash diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..c9a387e --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,17 @@ +version: 2 +workflows: + ci: + jobs: + - build_and_deploy: + filters: + branches: + only: debian + tags: + ignore: /.*/ +jobs: + build_and_deploy: + machine: + image: ubuntu-2004:202010-01 + steps: + - checkout + - run: tools/ci-bootstrap.bash diff --git a/Makefile b/Makefile index 7322f25..2b63505 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ export BUILD := build/$(T)/$(L) DEB := riju-$(T)-$(L).deb -S3_DEBS := s3://$(S3_BUCKET_BASE)-debs +S3_DEBS := s3://$(S3_BUCKET)-debs S3_DEB := $(S3_DEBS)/debs/$(DEB) S3_HASH := $(S3_DEBS)/hashes/riju-$(T)-$(L) @@ -38,7 +38,7 @@ endif script: @: $${L} $${T} mkdir -p $(BUILD) - node src/make-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash + node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash chmod +x $(BUILD)/build.bash .PHONY: pkg @@ -66,11 +66,11 @@ endif shell: @: $${I} ifeq ($(I),admin) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) --network host riju:$(I) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) --network host riju:$(I) $(CMD) else ifeq ($(I),compile) - docker run -it --rm --hostname $(I) $(SHELL_PORTS) riju:$(I) + docker run -it --rm --hostname $(I) $(SHELL_PORTS) riju:$(I) $(CMD) else - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) riju:$(I) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) riju:$(I) $(CMD) endif .PHONY: install @@ -116,13 +116,13 @@ dev: .PHONY: pull pull: - @: $${I} $${DOCKER_REPO_BASE} - docker pull $(DOCKER_REPO_BASE):$(I) - docker tag $(DOCKER_REPO_BASE):$(I) riju:$(I) + @: $${I} $${DOCKER_REPO} + docker pull $(DOCKER_REPO):$(I) + docker tag $(DOCKER_REPO):$(I) riju:$(I) .PHONY: download download: - @: $${L} $${T} $${S3_BUCKET_BASE} + @: $${L} $${T} $${S3_BUCKET} mkdir -p $(BUILD) aws s3 cp $(S3_DEB) $(BUILD)/$(DEB) @@ -130,15 +130,20 @@ download: .PHONY: push push: - @: $${I} $${DOCKER_REPO_BASE} - docker tag riju:$(I) $(DOCKER_REPO_BASE):$(I) - docker push $(DOCKER_REPO_BASE):$(I) + @: $${I} $${DOCKER_REPO} + docker tag riju:$(I) $(DOCKER_REPO):$(I) + docker push $(DOCKER_REPO):$(I) .PHONY: upload upload: - @: $${L} $${T} $${S3_BUCKET_BASE} + @: $${L} $${T} $${S3_BUCKET} + aws s3 rm --recursive $(S3_HASH) aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) - hash=$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash); test $${hash}; aws s3 cp - $(S3_HASH)/$${hash} < /dev/null + hash=$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash); test $${hash}; echo $${hash}; aws s3 cp - $(S3_HASH)/$${hash} < /dev/null + +.PHONY: publish +publish: + tools/publish.bash ### Miscellaneous diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 50d72ca..9d80b4c 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -31,6 +31,7 @@ EOF packages=" docker-ce-cli +git jq less make diff --git a/tf/infra.tf b/tf/infra.tf index c0fb08c..e6ac231 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -28,6 +28,44 @@ provider "aws" { data "aws_region" "current" {} +resource "aws_iam_user" "deploy" { + name = "riju-deploy" + tags = local.tags +} + +data "aws_iam_policy_document" "deploy" { + statement { + actions = [ + "s3:ListBucket", + ] + + resources = [ + "arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}", + ] + } + + statement { + actions = [ + "s3:*Object", + ] + + resources = [ + "arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}/*", + ] + } +} + +resource "aws_iam_policy" "deploy" { + name = "riju-deploy" + description = "Role used by CI to deploy Riju" + policy = data.aws_iam_policy_document.deploy.json +} + +resource "aws_iam_user_policy_attachment" "deploy" { + user = aws_iam_user.deploy.name + policy_arn = aws_iam_policy.deploy.arn +} + resource "aws_s3_bucket" "riju_debs" { bucket = "riju-debs" acl = "public-read" diff --git a/tools/build-composite-image.js b/tools/build-composite-image.js index 9f1b51d..17c0a99 100644 --- a/tools/build-composite-image.js +++ b/tools/build-composite-image.js @@ -6,6 +6,9 @@ import express from "express"; import { getLangs } from "./config.js"; import { runCommand } from "./util.js"; +// Get a Node.js http server object that will serve information and +// files for packages that should be installed into the composite +// Docker image. function getServer(langs) { const app = express(); app.get("/langs", (req, res) => { diff --git a/tools/ci-bootstrap.bash b/tools/ci-bootstrap.bash new file mode 100755 index 0000000..a3ba58c --- /dev/null +++ b/tools/ci-bootstrap.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +: ${AWS_ACCESS_KEY_ID} +: ${AWS_SECRET_ACCESS_KEY} +: ${DEPLOY_SSH_PRIVATE_KEY} +: ${DOCKER_PASSWORD} +: ${DOCKER_REPO} +: ${DOCKER_USERNAME} +: ${DOMAIN} +: ${S3_BUCKET} + +make pull image shell I=admin CMD="tools/ci-run.bash" diff --git a/tools/ci-run.bash b/tools/ci-run.bash new file mode 100755 index 0000000..2b9ab3d --- /dev/null +++ b/tools/ci-run.bash @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin +yarn install + +make publish diff --git a/tools/hash-composite-image.js b/tools/hash-composite-image.js new file mode 100644 index 0000000..c5fe2a3 --- /dev/null +++ b/tools/hash-composite-image.js @@ -0,0 +1,64 @@ +import crypto from "crypto"; +import process from "process"; + +import { getLangs } from "./config.js"; +import { runCommand } from "./util.js"; + +// Parse command-line arguments, run main functionality, and exit. +async function main() { + const args = process.argv.slice(2); + if (args.length !== 1) { + console.error("usage: node hash-composite-image.js (local | remote)"); + process.exit(1); + } + const mode = args[0]; + let getHash; + switch (mode) { + case "local": + getHash = async (lang, type) => { + return ( + await runCommand( + `dpkg-deb -f build/${type}/${lang}/riju-${type}-${lang}.deb Riju-Script-Hash`, + { getStdout: true } + ) + ).stdout.trim(); + }; + break; + case "remote": + const remoteHashes = Object.fromEntries( + ( + await runCommand("tools/list-s3-hashes.bash", { getStdout: true }) + ).stdout + .trim() + .split("\n") + .map((path) => { + const [_, pkg, hash] = path.split("/"); + return [pkg, hash]; + }) + ); + getHash = async (lang, type) => remoteHashes[`riju-${type}-${lang}`]; + break; + default: + console.error(`hash-composite-image.js: unsupported mode: ${mode}`); + process.exit(1); + } + const langs = await getLangs(); + const hashes = {}; + for (const lang of langs) { + for (const type of ["config", "lang"]) { + const hash = await getHash(lang, type); + if (hash.length !== 40) { + throw new Error(`malformed hash: ${hash}`); + } + hashes[`riju-${type}-${lang}`] = hash; + } + } + const allHashes = Object.values(hashes).sort().join(","); + console.log(crypto.createHash("sha1").update(allHashes).digest("hex")); + process.exit(0); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tools/list-s3-hashes.bash b/tools/list-s3-hashes.bash new file mode 100755 index 0000000..f05ff2d --- /dev/null +++ b/tools/list-s3-hashes.bash @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +aws s3api list-objects-v2 --bucket riju-debs --prefix hashes | jq -r '.Contents[].Key' diff --git a/tools/publish.bash b/tools/publish.bash new file mode 100755 index 0000000..4529e54 --- /dev/null +++ b/tools/publish.bash @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: ${DOCKER_REPO} +: ${DOMAIN} +: ${S3_BUCKET} + +if [[ -z "${DEPLOY_SSH_PRIVATE_KEY}" ]]; then + DEPLOY_SSH_PRIVATE_KEY="$(base64 < "${DEPLOY_SSH_PUBLIC_KEY_FILE%.pub}")" +fi + +make push I=admin +make pull image push I=packaging + +declare -A published_hashes +while read line; do + pkg="$(awk -F/ '{ print $2 }' <<< "${line}")" + hash="$(awk -F/ '{ print $3 }' <<< "${line}")" + published_hashes["${pkg}"]="${hash}" +done < <(tools/list-s3-hashes.bash) + +readarray -t langs < <(ls langs | grep '\.yaml$' | grep -Eo '^[^.]+') + +for lang in "${langs[@]}"; do + for type in lang config; do + make script L="${lang}" T="${type}" + done +done + +fmt='%-31s %-8s %s\n' +printf "${fmt}" "PACKAGE" "CHANGE" "HASH" +printf "${fmt}" "-------" "------" "----" + +declare -A local_hashes +for lang in "${langs[@]}"; do + for type in lang config; do + pkg="riju-${type}-${lang}" + hash="$(sha1sum "build/${type}/${lang}/build.bash" | awk '{ print $1 }')" + local_hashes["${pkg}"]="${hash}" + published_hash="${published_hashes["${pkg}"]:-}" + if [[ -z "${published_hash}" ]]; then + printf "${fmt}" "${pkg}" "create" "${hash}" + elif [[ "${published_hash}" != "${hash}" ]]; then + printf "${fmt}" "${pkg}" "update" "${published_hash} => ${hash}" + else + printf "${fmt}" "${pkg}" "" "${hash}" + fi + done +done + +if [[ -t 1 ]]; then + echo "Press enter to continue, or ctrl-C to abort..." + read +fi + +for lang in "${langs[@]}"; do + for type in lang config; do + pkg="riju-${type}-${lang}" + hash="${local_hashes["${pkg}"]}" + published_hash="${published_hashes["${pkg}"]:-}" + if [[ "${published_hash}" != "${hash}" ]]; then + make shell I=packaging CMD="make pkg L='${lang}' T='${type}'" + make upload L="${lang}" T="${type}" + fi + done +done + +composite_local_hash="$(node tools/hash-composite-image.js local)" +composite_remote_hash="$(node tools/hash-composite-image.js remote)" + +if [[ "${composite_local_hash}" != "${composite_remote_hash}" ]]; then + make image push I=composite +else + make pull I=composite +fi + +make pull image push I=app + +sha="$(git describe --match=always-omit-tag --always --abbrev=40 --dirty)" + +image="${DOCKER_REPO}:app-${sha}" + +docker tag "${DOCKER_REPO}:app" "${image}" +docker push "${image}" + +ssh -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i <(base64 -d <<< "${DEPLOY_SSH_PRIVATE_KEY}") \ + "deploy@${DOMAIN}" "${image}" diff --git a/tools/util.js b/tools/util.js index 5942c75..047c7ad 100644 --- a/tools/util.js +++ b/tools/util.js @@ -1,23 +1,45 @@ import child_process from "child_process"; +import process from "process"; -// Given a shell command as a string, execute it with Bash. -export async function runCommand(cmd) { +// Given a shell command as a string, execute it with Bash. Options: +// +// getStdout: if given and truthy, return stdout as a string instead +// of streaming it to the standard location +// getStderr: same but for stderr +// +// Return value is an object with required property code and, optional +// properties stdout, stderr. +export async function runCommand(cmd, options) { + const { getStdout, getStderr } = options || {}; console.error(`$ ${cmd}`); - return new Promise((resolve, reject) => { + const rv = await new Promise((resolve, reject) => { + const rv = { stdout: "", stderr: "" }; const proc = child_process.spawn( "bash", ["-c", `set -euo pipefail; ${cmd}`], { - stdio: "inherit", + stdio: [ + process.stdin, + getStdout ? "pipe" : process.stdout, + getStderr ? "pipe" : process.stderr, + ], } ); + if (getStdout) { + proc.stdout.on("data", (data) => { + rv.stdout += data.toString(); + }); + } + if (getStderr) { + proc.stderr.on("data", (data) => { + rv.stderr += data.toString(); + }); + } proc.on("error", reject); - proc.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`command exited with code ${code}`)); - } - }); + proc.on("close", (code) => resolve({ code, ...rv })); }); + if (rv.code !== 0) { + throw new Error(`command exited with code ${code}`); + } + return rv; }