From e466a94d18803718c96197d9a0d1f1da8483be13 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 7 Oct 2020 18:22:13 -0700 Subject: [PATCH] Migrate to Packer --- .circleci/config.yml | 8 +- .gitignore | 2 + Makefile | 4 +- packer/config-certbot.json | 22 +++++ packer/config.json | 55 ++++++++++++ packer/provision-certbot.bash | 11 +++ packer/provision.bash | 89 +++++++++++++++++++ .../resources}/certbot-post.bash | 0 .../resources}/certbot-pre.bash | 0 packer/resources/riju.bash | 31 +++++++ {scripts => packer/resources}/riju.service | 2 +- packer/resources/rijuctl.bash | 49 ++++++++++ scripts/deploy-phase1.py | 41 --------- scripts/deploy-phase2.py | 25 ------ scripts/deploy.bash | 53 ++++++++--- scripts/install-scripts.bash | 12 --- scripts/riju-serve.bash | 22 ----- 17 files changed, 309 insertions(+), 117 deletions(-) create mode 100644 packer/config-certbot.json create mode 100644 packer/config.json create mode 100755 packer/provision-certbot.bash create mode 100755 packer/provision.bash rename {scripts => packer/resources}/certbot-post.bash (100%) rename {scripts => packer/resources}/certbot-pre.bash (100%) create mode 100755 packer/resources/riju.bash rename {scripts => packer/resources}/riju.service (88%) create mode 100755 packer/resources/rijuctl.bash delete mode 100755 scripts/deploy-phase1.py delete mode 100755 scripts/deploy-phase2.py delete mode 100755 scripts/install-scripts.bash delete mode 100755 scripts/riju-serve.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index c4e90f3..915887a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,13 @@ jobs: - image: alpine steps: - checkout - - run: apk add --no-cache --no-progress bash openssh + - setup_remote_docker + - run: >- + apk add --no-cache --no-progress + bash docker make openssh + - run: >- + echo "${DOCKER_PASSWORD}" | + docker login --username "${DOCKER_USERNAME}" --password-stdin - run: scripts/deploy.bash workflows: version: 2 diff --git a/.gitignore b/.gitignore index cc99f2c..808bf7f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ *.log .log .lsp-repl-history +do_digitalocean.pem node_modules out +secrets.json tests tests-* diff --git a/Makefile b/Makefile index 46715d1..2ca3d2e 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,10 @@ image-prod: ## Build Docker image for production .PHONY: docker docker: image-dev docker-nobuild ## Run shell with source code and deps inside Docker -.PHONY: docker +.PHONY: docker-nobuild docker-nobuild: ## Same as 'make docker', but don't rebuild image scripts/docker.bash run -it --rm -v "$(PWD):/home/docker/src" -p 6119:6119 -p 6120:6120 -h riju riju bash .PHONY: deploy -deploy: ## Deploy current master from GitHub to production +deploy: image-prod ## Build, publish, and deploy production image scripts/deploy.bash diff --git a/packer/config-certbot.json b/packer/config-certbot.json new file mode 100644 index 0000000..da7bc53 --- /dev/null +++ b/packer/config-certbot.json @@ -0,0 +1,22 @@ +{ + "variables": { + "api_token": "" + }, + "sensitive-variables": ["api_token"], + "builders": [ + { + "type": "digitalocean", + "api_token": "{{user `api_token`}}", + "image": "ubuntu-20-04-x64", + "region": "sfo3", + "size": "s-1vcpu-1gb", + "ssh_username": "root" + } + ], + "provisioners": [ + { + "type": "shell", + "script": "provision-certbot.bash" + } + ] +} diff --git a/packer/config.json b/packer/config.json new file mode 100644 index 0000000..8e86a28 --- /dev/null +++ b/packer/config.json @@ -0,0 +1,55 @@ +{ + "variables": { + "api_token": "", + "admin_password": "", + "admin_ssh_public_key_file": "", + "deploy_ssh_public_key_file": "", + "docker_repo": "raxod502/riju" + }, + "sensitive-variables": ["api_token", "admin_password"], + "builders": [ + { + "type": "digitalocean", + "api_token": "{{user `api_token`}}", + "image": "ubuntu-20-04-x64", + "region": "sfo3", + "size": "s-1vcpu-1gb", + "ssh_username": "root" + } + ], + "provisioners": [ + { + "type": "file", + "source": "{{user `admin_ssh_public_key_file`}}", + "destination": "/tmp/id_admin.pub" + }, + { + "type": "file", + "source": "{{user `deploy_ssh_public_key_file`}}", + "destination": "/tmp/id_deploy.pub" + }, + { + "type": "file", + "source": "resources/riju.bash", + "destination": "/tmp/riju.bash" + }, + { + "type": "file", + "source": "resources/riju.service", + "destination": "/tmp/riju.service" + }, + { + "type": "file", + "source": "resources/rijuctl.bash", + "destination": "/tmp/rijuctl.bash" + }, + { + "type": "shell", + "script": "provision.bash", + "environment_vars": [ + "ADMIN_PASSWORD={{user `admin_password`}}", + "DOCKER_REPO={{user `docker_repo`}}" + ] + } + ] +} diff --git a/packer/provision-certbot.bash b/packer/provision-certbot.bash new file mode 100755 index 0000000..992665d --- /dev/null +++ b/packer/provision-certbot.bash @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get dist-upgrade -y + +apt-get install -y certbot + +rm -rf /var/lib/apt/lists/* diff --git a/packer/provision.bash b/packer/provision.bash new file mode 100755 index 0000000..12ebaa3 --- /dev/null +++ b/packer/provision.bash @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +if [[ -z "${ADMIN_PASSWORD}" ]]; then + echo "you need to set admin_password in secrets.json" >&2 + exit 1 +fi + +if [[ -z "${DOCKER_REPO}" ]]; then + echo "internal error: somehow DOCKER_REPO was not set" >&2 + exit 1 +fi + +for user in admin deploy; do + if [[ ! -s "/tmp/id_${user}.pub" ]]; then + echo "you need to set ${user}_ssh_public_key_file in secrets.json" >&2 + exit 1 + fi + + if ! grep -vq "PRIVATE KEY" "/tmp/id_${user}.pub"; then + echo "you accidentally set ${user}_ssh_public_key_file to a private key" >&2 + exit 1 + fi + + IFS=" " read contents < "/tmp/id_${user}.pub" + echo "${contents}" > "/tmp/id_${user}.pub" +done + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get dist-upgrade -y + +apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common +curl -sSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository -n universe +add-apt-repository -n "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + +packages=" + +bsdmainutils +certbot +containerd.io +docker-ce +docker-ce-cli +git +make +members +python3 +tmux +vim +whois + +" + +apt-get update +apt-get install -y ${packages} + +sed -Ei 's/^#?PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config +sed -Ei 's/^#?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config +sed -Ei 's/^#?PermitEmptyPasswords .*/PermitEmptyPasswords no/' /etc/ssh/sshd_config + +passwd -l root +useradd admin -g admin -G sudo -s /usr/bin/bash -p "$(echo "${ADMIN_PASSWORD}" | mkpasswd -s)" -m +useradd deploy -s /usr/bin/bash -p "!" + +for user in admin deploy; do + mkdir -p "/home/${user}/.ssh" + mv "/tmp/id_${user}.pub" "/home/${user}/.ssh/authorized_keys" + chown -R "${user}:${user}" "/home/${user}/.ssh" + chmod -R go-rwx "/home/${user}/.ssh" +done + +sed -i 's/^/command="sudo rijuctl",restrict/' /home/deploy/.ssh/authorized_keys + +cat <<"EOF" > /etc/sudoers.d/riju +deploy ALL=(root) NOPASSWD: /usr/local/bin/rijuctl +EOF + +sed -i "s#DOCKER_REPO_REPLACED_BY_PACKER#${DOCKER_REPO}#" /tmp/rijuctl.bash + +mv /tmp/riju.bash /usr/local/bin/riju +mv /tmp/riju.service /etc/systemd/system/riju.service +mv /tmp/rijuctl.bash /usr/local/bin/rijuctl + +chmod +x /usr/local/bin/riju +chmod +x /usr/local/bin/rijuctl + +rm -rf /var/lib/apt/lists/* diff --git a/scripts/certbot-post.bash b/packer/resources/certbot-post.bash similarity index 100% rename from scripts/certbot-post.bash rename to packer/resources/certbot-post.bash diff --git a/scripts/certbot-pre.bash b/packer/resources/certbot-pre.bash similarity index 100% rename from scripts/certbot-pre.bash rename to packer/resources/certbot-pre.bash diff --git a/packer/resources/riju.bash b/packer/resources/riju.bash new file mode 100755 index 0000000..c350863 --- /dev/null +++ b/packer/resources/riju.bash @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +domain="$(ls /etc/letsencrypt/live | grep -v README | head -n1)" + +if [[ -n "${domain}" ]]; then + echo "Detected cert for domain: ${domain}, enabling TLS" >&2 + export TLS=1 + TLS_PRIVATE_KEY="$(base64 "/etc/letsencrypt/live/${domain}/privkey.pem")" + TLS_CERTIFICATE="$(base64 "/etc/letsencrypt/live/${domain}/fullchain.pem")" + export TLS_PRIVATE_KEY TLS_CERTIFICATE + if [[ "${domain}" == riju.codes ]]; then + echo "Domain is riju.codes, enabling analytics" >&2 + export ANALYTICS=1 + else + echo "Domain is not riju.codes, disabling analytics" >&2 + fi +else + echo "No certs installed in /etc/letsencrypt/live, disabling TLS" >&2 +fi + +if [[ -t 1 ]]; then + it=-it +else + it= +fi + +docker run ${it} -e TLS -e TLS_PRIVATE_KEY -e TLS_CERTIFICATE -e ANALYTICS \ + --rm -p 0.0.0.0:80:6119 -p 0.0.0.0:443:6120 -h riju riju:live diff --git a/scripts/riju.service b/packer/resources/riju.service similarity index 88% rename from scripts/riju.service rename to packer/resources/riju.service index 2854d1f..6ea3ac0 100644 --- a/scripts/riju.service +++ b/packer/resources/riju.service @@ -5,7 +5,7 @@ After=docker.service [Service] Type=exec -ExecStart=riju-serve +ExecStart=riju Restart=always [Install] diff --git a/packer/resources/rijuctl.bash b/packer/resources/rijuctl.bash new file mode 100755 index 0000000..9add265 --- /dev/null +++ b/packer/resources/rijuctl.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +DOCKER_REPO="${DOCKER_REPO:-DOCKER_REPO_REPLACED_BY_PACKER}" + +if [[ -n "${SSH_ORIGINAL_COMMAND}" ]]; then + set -- ${SSH_ORIGINAL_COMMAND} +fi + +function usage { + echo "usage: rijuctl deploy TAG" >&2 + exit 1 +} + +function main { + if (( $# == 0 )); then + usage + fi + + subcmd="$1" + shift + case "${subcmd}" in + deploy) + deploy "$@" + ;; + *) + usage + ;; + esac +} + +function deploy { + if (( $# != 1 )); then + usage + fi + tag="$1" + if [[ -z "${tag}" ]]; then + usage + fi + + docker pull "${DOCKER_REPO}:${tag}" + docker tag riju:live riju:prev + docker tag "${DOCKER_REPO}:${tag}" riju:live + docker system prune -f + systemctl restart riju +} + +main "$@" diff --git a/scripts/deploy-phase1.py b/scripts/deploy-phase1.py deleted file mode 100755 index be09219..0000000 --- a/scripts/deploy-phase1.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import errno -import os -import re -import signal -import subprocess -import sys -import tempfile -import time - -result = subprocess.run(["pgrep", "-x", "riju-deploy"], stdout=subprocess.PIPE) -assert result.returncode in {0, 1} -for pid in result.stdout.decode().splitlines(): - print(f"Found existing process {pid}, trying to kill ...", file=sys.stderr) - pid = int(pid) - os.kill(pid, signal.SIGTERM) - while True: - time.sleep(0.01) - try: - os.kill(pid, 0) - except OSError as e: - if e.errno == errno.ESRCH: - break - -with tempfile.TemporaryDirectory() as tmpdir: - os.chdir(tmpdir) - subprocess.run( - [ - "git", - "clone", - "https://github.com/raxod502/riju.git", - "--single-branch", - "--depth=1", - "--no-tags", - ], - check=True, - ) - os.chdir("riju") - subprocess.run(["scripts/deploy-phase2.py"], check=True) diff --git a/scripts/deploy-phase2.py b/scripts/deploy-phase2.py deleted file mode 100755 index afd2bba..0000000 --- a/scripts/deploy-phase2.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import errno -import os -import re -import signal -import subprocess -import sys -import tempfile -import time - -subprocess.run(["docker", "pull", "ubuntu:rolling"], check=True) -subprocess.run(["docker", "system", "prune", "-f"], check=True) -subprocess.run(["make", "image-prod"], check=True) -existing_containers = subprocess.run( - ["docker", "ps", "-q"], check=True, stdout=subprocess.PIPE -).stdout.splitlines() -subprocess.run(["scripts/install-scripts.bash"], check=True) -if existing_containers: - subprocess.run(["docker", "kill", *existing_containers], check=True) -subprocess.run(["systemctl", "enable", "riju"], check=True) -subprocess.run(["systemctl", "restart", "riju"], check=True) - -print("==> Successfully deployed Riju! <==", file=sys.stderr) diff --git a/scripts/deploy.bash b/scripts/deploy.bash index c70c13b..b03b810 100755 --- a/scripts/deploy.bash +++ b/scripts/deploy.bash @@ -1,23 +1,50 @@ #!/usr/bin/env bash -set -e -set -o pipefail +set -euxo pipefail -tmpdir="$(mktemp -d)" -keyfile="${tmpdir}/id" - -if [[ -n "$DEPLOY_KEY" ]]; then - printf '%s\n' "$DEPLOY_KEY" | base64 -d > "$keyfile" -elif [[ -f "$HOME/.ssh/id_rsa_riju_deploy" ]]; then - cp "$HOME/.ssh/id_rsa_riju_deploy" "$keyfile" -else - echo 'deploy.bash: you must set $DEPLOY_KEY' >&2 +if [[ -z "${DOCKER_REPO}" ]]; then + echo "environment variable not set: DOCKER_REPO" >&2 exit 1 fi -chmod go-rw "$keyfile" +if [[ -z "${DOMAIN}" ]]; then + echo "environment variable not set: DOMAIN" >&2 + exit 1 +fi + +if [[ -z "${DEPLOY_SSH_PRIVATE_KEY}" ]]; then + if [[ -f "$HOME/.ssh/id_rsa_riju_deploy" ]]; then + DEPLOY_SSH_PRIVATE_KEY="$(< "$HOME/.ssh/id_rsa_riju_deploy")" + else + echo "environment variable not set: DEPLOY_SSH_PRIVATE_KEY" + fi +else + DEPLOY_SSH_PRIVATE_KEY="$(printf '%s\n' "${DEPLOY_SSH_PRIVATE_KEY}" | base64 -d)" +fi + +tag="$(date +%s%3N)-$(git branch --show-current)-$(git rev-parse @)" + +if [[ -n "$(git status --porcelain)" ]]; then + tag="${tag}-dirty" +fi + +scripts/docker.bash tag riju:prod "${DOCKER_REPO}:${tag}" +scripts/docker.bash push "${DOCKER_REPO}:${tag}" + +tmpdir="$(mktemp -d)" + +function cleanup { + rm -rf "${tmpdir}" +} + +trap cleanup EXIT + +printf '%s' "${DEPLOY_SSH_PRIVATE_KEY}" > "${tmpdir}/id" + +chmod go-rw "${tmpdir}/id" ssh -o IdentitiesOnly=yes \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -o LogLevel=QUIET \ - -i "${keyfile}" deploy@riju.codes + -i "${tmpdir}/id" "deploy@${DOMAIN}" \ + deploy "${tag}" diff --git a/scripts/install-scripts.bash b/scripts/install-scripts.bash deleted file mode 100755 index 494a92e..0000000 --- a/scripts/install-scripts.bash +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -cp scripts/riju.service /etc/systemd/system/riju.service -cp scripts/riju-serve.bash /usr/local/bin/riju-serve -cp scripts/certbot-pre.bash /etc/letsencrypt/renewal-hooks/pre/riju -cp scripts/certbot-post.bash /etc/letsencrypt/renewal-hooks/post/riju -cp scripts/deploy-phase1.py /usr/local/bin/riju-deploy - -systemctl daemon-reload diff --git a/scripts/riju-serve.bash b/scripts/riju-serve.bash deleted file mode 100755 index 2c45189..0000000 --- a/scripts/riju-serve.bash +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -TLS=1 -TLS_PRIVATE_KEY="$(base64 /etc/letsencrypt/live/riju.codes/privkey.pem)" -TLS_CERTIFICATE="$(base64 /etc/letsencrypt/live/riju.codes/fullchain.pem)" -ANALYTICS=1 - -# Do this separately so that errors in command substitution will crash -# the script. -export TLS TLS_PRIVATE_KEY TLS_CERTIFICATE ANALYTICS - -if [[ -t 1 ]]; then - it=-it -else - it= -fi - -docker run ${it} -e TLS -e TLS_PRIVATE_KEY -e TLS_CERTIFICATE -e ANALYTICS \ - --rm -p 0.0.0.0:80:6119 -p 0.0.0.0:443:6120 -h riju riju:prod