Set up initial CI configuration
This commit is contained in:
Radon Rosborough
parent
57e28ea1e2
commit
77387027f2
|
|
@ -0,0 +1,17 @@
|
|||
version: 2
|
||||
workflows:
|
||||
ci:
|
||||
jobs:
|
||||
- build_and_deploy:
|
||||
filters:
|
||||
branches:
|
||||
only: debian
|
||||
tags:
|
||||
ignore: /.*/
|
||||
jobs:
|
||||
build_and_deploy:
|
||||
machine:
|
||||
image: ubuntu-2004:202010-01
|
||||
steps:
|
||||
- checkout
|
||||
- run: tools/ci-bootstrap.bash
|
||||
33
Makefile
33
Makefile
|
|
@ -8,7 +8,7 @@ export
|
|||
|
||||
BUILD := build/$(T)/$(L)
|
||||
DEB := riju-$(T)-$(L).deb
|
||||
S3_DEBS := s3://$(S3_BUCKET_BASE)-debs
|
||||
S3_DEBS := s3://$(S3_BUCKET)-debs
|
||||
S3_DEB := $(S3_DEBS)/debs/$(DEB)
|
||||
S3_HASH := $(S3_DEBS)/hashes/riju-$(T)-$(L)
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ endif
|
|||
script:
|
||||
@: $${L} $${T}
|
||||
mkdir -p $(BUILD)
|
||||
node src/make-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash
|
||||
node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash
|
||||
chmod +x $(BUILD)/build.bash
|
||||
|
||||
.PHONY: pkg
|
||||
|
|
@ -66,11 +66,11 @@ endif
|
|||
shell:
|
||||
@: $${I}
|
||||
ifeq ($(I),admin)
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) --network host riju:$(I)
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) --network host riju:$(I) $(CMD)
|
||||
else ifeq ($(I),compile)
|
||||
docker run -it --rm --hostname $(I) $(SHELL_PORTS) riju:$(I)
|
||||
docker run -it --rm --hostname $(I) $(SHELL_PORTS) riju:$(I) $(CMD)
|
||||
else
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) riju:$(I)
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) riju:$(I) $(CMD)
|
||||
endif
|
||||
|
||||
.PHONY: install
|
||||
|
|
@ -116,13 +116,13 @@ dev:
|
|||
|
||||
.PHONY: pull
|
||||
pull:
|
||||
@: $${I} $${DOCKER_REPO_BASE}
|
||||
docker pull $(DOCKER_REPO_BASE):$(I)
|
||||
docker tag $(DOCKER_REPO_BASE):$(I) riju:$(I)
|
||||
@: $${I} $${DOCKER_REPO}
|
||||
docker pull $(DOCKER_REPO):$(I)
|
||||
docker tag $(DOCKER_REPO):$(I) riju:$(I)
|
||||
|
||||
.PHONY: download
|
||||
download:
|
||||
@: $${L} $${T} $${S3_BUCKET_BASE}
|
||||
@: $${L} $${T} $${S3_BUCKET}
|
||||
mkdir -p $(BUILD)
|
||||
aws s3 cp $(S3_DEB) $(BUILD)/$(DEB)
|
||||
|
||||
|
|
@ -130,15 +130,20 @@ download:
|
|||
|
||||
.PHONY: push
|
||||
push:
|
||||
@: $${I} $${DOCKER_REPO_BASE}
|
||||
docker tag riju:$(I) $(DOCKER_REPO_BASE):$(I)
|
||||
docker push $(DOCKER_REPO_BASE):$(I)
|
||||
@: $${I} $${DOCKER_REPO}
|
||||
docker tag riju:$(I) $(DOCKER_REPO):$(I)
|
||||
docker push $(DOCKER_REPO):$(I)
|
||||
|
||||
.PHONY: upload
|
||||
upload:
|
||||
@: $${L} $${T} $${S3_BUCKET_BASE}
|
||||
@: $${L} $${T} $${S3_BUCKET}
|
||||
aws s3 rm --recursive $(S3_HASH)
|
||||
aws s3 cp $(BUILD)/$(DEB) $(S3_DEB)
|
||||
hash=$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash); test $${hash}; aws s3 cp - $(S3_HASH)/$${hash} < /dev/null
|
||||
hash=$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash); test $${hash}; echo $${hash}; aws s3 cp - $(S3_HASH)/$${hash} < /dev/null
|
||||
|
||||
.PHONY: publish
|
||||
publish:
|
||||
tools/publish.bash
|
||||
|
||||
### Miscellaneous
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ EOF
|
|||
packages="
|
||||
|
||||
docker-ce-cli
|
||||
git
|
||||
jq
|
||||
less
|
||||
make
|
||||
|
|
|
|||
38
tf/infra.tf
38
tf/infra.tf
|
|
@ -28,6 +28,44 @@ provider "aws" {
|
|||
|
||||
data "aws_region" "current" {}
|
||||
|
||||
resource "aws_iam_user" "deploy" {
|
||||
name = "riju-deploy"
|
||||
tags = local.tags
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "deploy" {
|
||||
statement {
|
||||
actions = [
|
||||
"s3:ListBucket",
|
||||
]
|
||||
|
||||
resources = [
|
||||
"arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}",
|
||||
]
|
||||
}
|
||||
|
||||
statement {
|
||||
actions = [
|
||||
"s3:*Object",
|
||||
]
|
||||
|
||||
resources = [
|
||||
"arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}/*",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "deploy" {
|
||||
name = "riju-deploy"
|
||||
description = "Role used by CI to deploy Riju"
|
||||
policy = data.aws_iam_policy_document.deploy.json
|
||||
}
|
||||
|
||||
resource "aws_iam_user_policy_attachment" "deploy" {
|
||||
user = aws_iam_user.deploy.name
|
||||
policy_arn = aws_iam_policy.deploy.arn
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket" "riju_debs" {
|
||||
bucket = "riju-debs"
|
||||
acl = "public-read"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ import express from "express";
|
|||
import { getLangs } from "./config.js";
|
||||
import { runCommand } from "./util.js";
|
||||
|
||||
// Get a Node.js http server object that will serve information and
|
||||
// files for packages that should be installed into the composite
|
||||
// Docker image.
|
||||
function getServer(langs) {
|
||||
const app = express();
|
||||
app.get("/langs", (req, res) => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
: ${AWS_ACCESS_KEY_ID}
|
||||
: ${AWS_SECRET_ACCESS_KEY}
|
||||
: ${DEPLOY_SSH_PRIVATE_KEY}
|
||||
: ${DOCKER_PASSWORD}
|
||||
: ${DOCKER_REPO}
|
||||
: ${DOCKER_USERNAME}
|
||||
: ${DOMAIN}
|
||||
: ${S3_BUCKET}
|
||||
|
||||
make pull image shell I=admin CMD="tools/ci-run.bash"
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin
|
||||
yarn install
|
||||
|
||||
make publish
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import crypto from "crypto";
|
||||
import process from "process";
|
||||
|
||||
import { getLangs } from "./config.js";
|
||||
import { runCommand } from "./util.js";
|
||||
|
||||
// Parse command-line arguments, run main functionality, and exit.
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length !== 1) {
|
||||
console.error("usage: node hash-composite-image.js (local | remote)");
|
||||
process.exit(1);
|
||||
}
|
||||
const mode = args[0];
|
||||
let getHash;
|
||||
switch (mode) {
|
||||
case "local":
|
||||
getHash = async (lang, type) => {
|
||||
return (
|
||||
await runCommand(
|
||||
`dpkg-deb -f build/${type}/${lang}/riju-${type}-${lang}.deb Riju-Script-Hash`,
|
||||
{ getStdout: true }
|
||||
)
|
||||
).stdout.trim();
|
||||
};
|
||||
break;
|
||||
case "remote":
|
||||
const remoteHashes = Object.fromEntries(
|
||||
(
|
||||
await runCommand("tools/list-s3-hashes.bash", { getStdout: true })
|
||||
).stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((path) => {
|
||||
const [_, pkg, hash] = path.split("/");
|
||||
return [pkg, hash];
|
||||
})
|
||||
);
|
||||
getHash = async (lang, type) => remoteHashes[`riju-${type}-${lang}`];
|
||||
break;
|
||||
default:
|
||||
console.error(`hash-composite-image.js: unsupported mode: ${mode}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const langs = await getLangs();
|
||||
const hashes = {};
|
||||
for (const lang of langs) {
|
||||
for (const type of ["config", "lang"]) {
|
||||
const hash = await getHash(lang, type);
|
||||
if (hash.length !== 40) {
|
||||
throw new Error(`malformed hash: ${hash}`);
|
||||
}
|
||||
hashes[`riju-${type}-${lang}`] = hash;
|
||||
}
|
||||
}
|
||||
const allHashes = Object.values(hashes).sort().join(",");
|
||||
console.log(crypto.createHash("sha1").update(allHashes).digest("hex"));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
aws s3api list-objects-v2 --bucket riju-debs --prefix hashes | jq -r '.Contents[].Key'
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: ${DOCKER_REPO}
|
||||
: ${DOMAIN}
|
||||
: ${S3_BUCKET}
|
||||
|
||||
if [[ -z "${DEPLOY_SSH_PRIVATE_KEY}" ]]; then
|
||||
DEPLOY_SSH_PRIVATE_KEY="$(base64 < "${DEPLOY_SSH_PUBLIC_KEY_FILE%.pub}")"
|
||||
fi
|
||||
|
||||
make push I=admin
|
||||
make pull image push I=packaging
|
||||
|
||||
declare -A published_hashes
|
||||
while read line; do
|
||||
pkg="$(awk -F/ '{ print $2 }' <<< "${line}")"
|
||||
hash="$(awk -F/ '{ print $3 }' <<< "${line}")"
|
||||
published_hashes["${pkg}"]="${hash}"
|
||||
done < <(tools/list-s3-hashes.bash)
|
||||
|
||||
readarray -t langs < <(ls langs | grep '\.yaml$' | grep -Eo '^[^.]+')
|
||||
|
||||
for lang in "${langs[@]}"; do
|
||||
for type in lang config; do
|
||||
make script L="${lang}" T="${type}"
|
||||
done
|
||||
done
|
||||
|
||||
fmt='%-31s %-8s %s\n'
|
||||
printf "${fmt}" "PACKAGE" "CHANGE" "HASH"
|
||||
printf "${fmt}" "-------" "------" "----"
|
||||
|
||||
declare -A local_hashes
|
||||
for lang in "${langs[@]}"; do
|
||||
for type in lang config; do
|
||||
pkg="riju-${type}-${lang}"
|
||||
hash="$(sha1sum "build/${type}/${lang}/build.bash" | awk '{ print $1 }')"
|
||||
local_hashes["${pkg}"]="${hash}"
|
||||
published_hash="${published_hashes["${pkg}"]:-}"
|
||||
if [[ -z "${published_hash}" ]]; then
|
||||
printf "${fmt}" "${pkg}" "create" "${hash}"
|
||||
elif [[ "${published_hash}" != "${hash}" ]]; then
|
||||
printf "${fmt}" "${pkg}" "update" "${published_hash} => ${hash}"
|
||||
else
|
||||
printf "${fmt}" "${pkg}" "" "${hash}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [[ -t 1 ]]; then
|
||||
echo "Press enter to continue, or ctrl-C to abort..."
|
||||
read
|
||||
fi
|
||||
|
||||
for lang in "${langs[@]}"; do
|
||||
for type in lang config; do
|
||||
pkg="riju-${type}-${lang}"
|
||||
hash="${local_hashes["${pkg}"]}"
|
||||
published_hash="${published_hashes["${pkg}"]:-}"
|
||||
if [[ "${published_hash}" != "${hash}" ]]; then
|
||||
make shell I=packaging CMD="make pkg L='${lang}' T='${type}'"
|
||||
make upload L="${lang}" T="${type}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
composite_local_hash="$(node tools/hash-composite-image.js local)"
|
||||
composite_remote_hash="$(node tools/hash-composite-image.js remote)"
|
||||
|
||||
if [[ "${composite_local_hash}" != "${composite_remote_hash}" ]]; then
|
||||
make image push I=composite
|
||||
else
|
||||
make pull I=composite
|
||||
fi
|
||||
|
||||
make pull image push I=app
|
||||
|
||||
sha="$(git describe --match=always-omit-tag --always --abbrev=40 --dirty)"
|
||||
|
||||
image="${DOCKER_REPO}:app-${sha}"
|
||||
|
||||
docker tag "${DOCKER_REPO}:app" "${image}"
|
||||
docker push "${image}"
|
||||
|
||||
ssh -o IdentitiesOnly=yes \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-i <(base64 -d <<< "${DEPLOY_SSH_PRIVATE_KEY}") \
|
||||
"deploy@${DOMAIN}" "${image}"
|
||||
|
|
@ -1,23 +1,45 @@
|
|||
import child_process from "child_process";
|
||||
import process from "process";
|
||||
|
||||
// Given a shell command as a string, execute it with Bash.
|
||||
export async function runCommand(cmd) {
|
||||
// Given a shell command as a string, execute it with Bash. Options:
|
||||
//
|
||||
// getStdout: if given and truthy, return stdout as a string instead
|
||||
// of streaming it to the standard location
|
||||
// getStderr: same but for stderr
|
||||
//
|
||||
// Return value is an object with required property code and, optional
|
||||
// properties stdout, stderr.
|
||||
export async function runCommand(cmd, options) {
|
||||
const { getStdout, getStderr } = options || {};
|
||||
console.error(`$ ${cmd}`);
|
||||
return new Promise((resolve, reject) => {
|
||||
const rv = await new Promise((resolve, reject) => {
|
||||
const rv = { stdout: "", stderr: "" };
|
||||
const proc = child_process.spawn(
|
||||
"bash",
|
||||
["-c", `set -euo pipefail; ${cmd}`],
|
||||
{
|
||||
stdio: "inherit",
|
||||
stdio: [
|
||||
process.stdin,
|
||||
getStdout ? "pipe" : process.stdout,
|
||||
getStderr ? "pipe" : process.stderr,
|
||||
],
|
||||
}
|
||||
);
|
||||
if (getStdout) {
|
||||
proc.stdout.on("data", (data) => {
|
||||
rv.stdout += data.toString();
|
||||
});
|
||||
}
|
||||
if (getStderr) {
|
||||
proc.stderr.on("data", (data) => {
|
||||
rv.stderr += data.toString();
|
||||
});
|
||||
}
|
||||
proc.on("error", reject);
|
||||
proc.on("close", (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`command exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
proc.on("close", (code) => resolve({ code, ...rv }));
|
||||
});
|
||||
if (rv.code !== 0) {
|
||||
throw new Error(`command exited with code ${code}`);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue