[#80] Various updates to run CI on EC2 dynamic runner

Not quite working yet but merging it now so I can focus on some other
higher-priority things.

.github/workflows/main.yml

@@ -17,7 +17,4 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_REGION: us-west-1
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          DOCKER_REPO: 084011155226.dkr.ecr.us-west-1.amazonaws.com/riju
+        run: tools/ci-ec2.bash
-          PUBLIC_DOCKER_REPO: public.ecr.aws/raxod502/riju
-          S3_BUCKET: riju
-        run: tools/ci-bootstrap.bash

Makefile

@@ -276,8 +276,11 @@ fmt: fmt-c fmt-go fmt-python fmt-terraform fmt-web # Format all code
 
 ### Infrastructure
 
-packer: supervisor # Build and publish a new AMI
+packer-web: supervisor # Build and publish a new webserver AMI
-	tools/packer-build.bash
+	tools/packer-build-web.bash
+
+packer-ci: # Build and publish a new CI AMI
+	tools/packer-build-ci.bash
 
 ### Miscellaneous
 

packer/ci.pkr.hcl

@@ -0,0 +1,48 @@
+data "amazon-ami" "ubuntu" {
+  filters = {
+    name                = "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*"
+    root-device-type    = "ebs"
+    virtualization-type = "hvm"
+  }
+  most_recent = true
+  owners      = ["099720109477"]
+}
+
+locals {
+  timestamp = regex_replace(timestamp(), "[- TZ:]", "")
+}
+
+source "amazon-ebs" "ubuntu" {
+  ami_name      = "riju-ci-${local.timestamp}"
+  instance_type = "t3.micro"
+  source_ami    = "${data.amazon-ami.ubuntu.id}"
+  ssh_username  = "ubuntu"
+
+  tag {
+    key = "BillingCategory"
+    value = "Riju"
+  }
+
+  tag {
+    key = "BillingSubcategory"
+    value = "Riju:AMI"
+  }
+
+  tag {
+    key = "Name"
+    value = "riju-ci-${local.timestamp}"
+  }
+}
+
+build {
+  sources = ["source.amazon-ebs.ubuntu"]
+
+  provisioner "file" {
+    destination = "/tmp/riju-init-volume"
+    source      = "riju-init-volume"
+  }
+
+  provisioner "shell" {
+    script = "provision-ci.bash"
+  }
+}

packer/provision-ci.bash

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# I think there is a race condition related to Ubuntu wanting to do an
+# automated system upgrade at boot, which causes 'apt-get update' to
+# sometimes fail with an obscure error message.
+sleep 5
+
+mkdir /tmp/riju-work
+pushd /tmp/riju-work
+
+export DEBIAN_FRONTEND=noninteractive
+
+sudo -E apt-get update
+sudo -E apt-get dist-upgrade -y
+
+sudo -E apt-get install -y curl gnupg lsb-release
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add -
+
+ubuntu_name="$(lsb_release -cs)"
+
+sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null <<EOF
+deb [arch=amd64] https://download.docker.com/linux/ubuntu ${ubuntu_name} stable
+EOF
+
+sudo -E apt-get update
+sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io make
+
+sudo chown root:root /tmp/riju-init-volume
+sudo mv /tmp/riju-init-volume /usr/local/bin/
+
+popd
+rm -rf /tmp/riju-work

packer/riju-init-volume

@@ -6,7 +6,7 @@ function print {
     echo >&2 "riju-init-volume: $@"
 }
 
-mount_point=/mnt/riju/data
+mount_point=/mnt/riju
 
 mkdir -p "${mount_point}"
 
@@ -55,7 +55,7 @@ mount -a
 
 print "filesystem mounted at ${mount_point}"
 
-docker_args="-g ${mount_point}"
+docker_args="-g ${mount_point}/docker"
 
 if ! cat /lib/systemd/system/docker.service | grep -q -- "${docker_args}"; then
     print "adding '${docker_args}' to docker.service"

packer/web.pkr.hcl

@@ -33,7 +33,7 @@ locals {
 }
 
 source "amazon-ebs" "ubuntu" {
-  ami_name      = "riju-${local.timestamp}"
+  ami_name      = "riju-web-${local.timestamp}"
   instance_type = "t3.micro"
   source_ami    = "${data.amazon-ami.ubuntu.id}"
   ssh_username  = "ubuntu"
@@ -50,7 +50,7 @@ source "amazon-ebs" "ubuntu" {
 
   tag {
     key = "Name"
-    value = "riju-${local.timestamp}"
+    value = "riju-web-${local.timestamp}"
   }
 }
 
@@ -84,6 +84,6 @@ build {
       "S3_BUCKET=${var.s3_bucket}",
       "SUPERVISOR_ACCESS_TOKEN=${var.supervisor_access_token}",
     ]
-    script           = "provision.bash"
+    script           = "provision-web.bash"
   }
 }

tf/alb.tf

@@ -74,8 +74,6 @@ resource "aws_lb_listener" "server_https" {
 }
 
 resource "aws_autoscaling_attachment" "server" {
-  count = local.ami_available ? 1 : 0
+  autoscaling_group_name = aws_autoscaling_group.server.name
-
-  autoscaling_group_name = aws_autoscaling_group.server[count.index].name
   alb_target_group_arn   = aws_lb_target_group.server.arn
 }

tf/ami.tf

@@ -1,6 +1,4 @@
 data "aws_ami" "server" {
-  count = local.ami_available ? 1 : 0
-
   owners = ["self"]
 
   filter {
@@ -8,3 +6,12 @@ data "aws_ami" "server" {
     values = [data.external.env.result.AMI_NAME]
   }
 }
+
+data "aws_ami" "ci" {
+  owners = ["self"]
+
+  filter {
+    name   = "name"
+    values = [data.external.env.result.CI_AMI_NAME]
+  }
+}

tf/asg.tf

@@ -35,10 +35,8 @@ resource "aws_security_group" "server" {
 }
 
 resource "aws_launch_template" "server" {
-  count = local.ami_available ? 1 : 0
-
   name          = "riju-server"
-  image_id      = data.aws_ami.server[count.index].id
+  image_id      = data.aws_ami.server.id
   instance_type = "t3.small"
 
   security_group_names = [aws_security_group.server.name]
@@ -78,8 +76,6 @@ resource "aws_launch_template" "server" {
 }
 
 resource "aws_autoscaling_group" "server" {
-  count = local.ami_available ? 1 : 0
-
   name = "riju-server"
 
   availability_zones = [
@@ -90,7 +86,7 @@ resource "aws_autoscaling_group" "server" {
   max_size         = 3
 
   launch_template {
-    id = aws_launch_template.server[count.index].id
+    id = aws_launch_template.server.id
   }
 
   tags = concat(

tf/cloudwatch.tf

@@ -1,20 +1,19 @@
 resource "aws_cloudwatch_metric_alarm" "server_cpu" {
-  count = local.ami_available ? 1 : 0
-
   alarm_name                = "riju-server-cpu-high"
   comparison_operator       = "GreaterThanOrEqualToThreshold"
   evaluation_periods        = "30"
+  datapoints_to_alarm       = "15"
   metric_name               = "cpu_usage_active"
   namespace                 = "CWAgent"
   period                    = "60"
-  statistic                 = "Maximum"
+  statistic                 = "Average"
-  threshold                 = "90"
+  threshold                 = "70"
-  alarm_description         = "CPU usage on Riju server is above 90% for 30 minutes"
+  alarm_description         = "Average CPU usage on Riju server is above 70% for 30 minutes"
   ok_actions                = [aws_sns_topic.riju.arn]
   alarm_actions             = [aws_sns_topic.riju.arn]
   insufficient_data_actions = [aws_sns_topic.riju.arn]
   dimensions = {
-    AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
+    AutoScalingGroupName = aws_autoscaling_group.server.name
   }
 
   tags = {
@@ -23,22 +22,21 @@ resource "aws_cloudwatch_metric_alarm" "server_cpu" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "server_memory" {
-  count = local.ami_available ? 1 : 0
-
   alarm_name                = "riju-server-memory-high"
   comparison_operator       = "GreaterThanOrEqualToThreshold"
   evaluation_periods        = "30"
+  datapoints_to_alarm       = "15"
   metric_name               = "mem_used_percent"
   namespace                 = "CWAgent"
   period                    = "60"
-  statistic                 = "Maximum"
+  statistic                 = "Average"
-  threshold                 = "80"
+  threshold                 = "70"
-  alarm_description         = "Memory usage on Riju server is above 80% for 30 minutes"
+  alarm_description         = "Average memory usage on Riju server is above 70% for 30 minutes"
   ok_actions                = [aws_sns_topic.riju.arn]
   alarm_actions             = [aws_sns_topic.riju.arn]
   insufficient_data_actions = [aws_sns_topic.riju.arn]
   dimensions = {
-    AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
+    AutoScalingGroupName = aws_autoscaling_group.server.name
   }
 
   tags = {
@@ -47,22 +45,21 @@ resource "aws_cloudwatch_metric_alarm" "server_memory" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "server_data_volume_disk_space" {
-  count = local.ami_available ? 1 : 0
-
   alarm_name                = "riju-server-data-volume-disk-usage-high"
   comparison_operator       = "GreaterThanOrEqualToThreshold"
-  evaluation_periods        = "30"
+  evaluation_periods        = "5"
+  datapoints_to_alarm       = "5"
   metric_name               = "disk_used_percent"
   namespace                 = "CWAgent"
   period                    = "60"
-  statistic                 = "Maximum"
+  statistic                 = "Average"
-  threshold                 = "90"
+  threshold                 = "70"
-  alarm_description         = "Disk space usage for data volume on Riju server is above 90% for 30 minutes"
+  alarm_description         = "Disk space usage for data volume on Riju server is above 70%"
   ok_actions                = [aws_sns_topic.riju.arn]
   alarm_actions             = [aws_sns_topic.riju.arn]
   insufficient_data_actions = [aws_sns_topic.riju.arn]
   dimensions = {
-    AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
+    AutoScalingGroupName = aws_autoscaling_group.server.name
     path                 = "/mnt/riju/data"
   }
 
@@ -72,22 +69,21 @@ resource "aws_cloudwatch_metric_alarm" "server_data_volume_disk_space" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "server_root_volume_disk_space" {
-  count = local.ami_available ? 1 : 0
-
   alarm_name                = "riju-server-root-volume-disk-usage-high"
   comparison_operator       = "GreaterThanOrEqualToThreshold"
-  evaluation_periods        = "30"
+  evaluation_periods        = "5"
+  datapoints_to_alarm       = "5"
   metric_name               = "disk_used_percent"
   namespace                 = "CWAgent"
   period                    = "60"
-  statistic                 = "Maximum"
+  statistic                 = "Average"
-  threshold                 = "90"
+  threshold                 = "70"
-  alarm_description         = "Disk space usage for root volume on Riju server is above 90% for 30 minutes"
+  alarm_description         = "Disk space usage for root volume on Riju server is above 70%"
   ok_actions                = [aws_sns_topic.riju.arn]
   alarm_actions             = [aws_sns_topic.riju.arn]
   insufficient_data_actions = [aws_sns_topic.riju.arn]
   dimensions = {
-    AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
+    AutoScalingGroupName = aws_autoscaling_group.server.name
     path                 = "/"
   }
 
@@ -97,8 +93,6 @@ resource "aws_cloudwatch_metric_alarm" "server_root_volume_disk_space" {
 }
 
 resource "aws_cloudwatch_dashboard" "riju" {
-  count = local.ami_available ? 1 : 0
-
   dashboard_name = "Riju"
   dashboard_body = <<EOF
 {
@@ -113,7 +107,7 @@ resource "aws_cloudwatch_dashboard" "riju" {
                 "title": "CPU",
                 "annotations": {
                     "alarms": [
-                        "${aws_cloudwatch_metric_alarm.server_cpu[count.index].arn}"
+                        "${aws_cloudwatch_metric_alarm.server_cpu.arn}"
                     ]
                 },
                 "view": "timeSeries",
@@ -130,7 +124,7 @@ resource "aws_cloudwatch_dashboard" "riju" {
                 "title": "Root volume disk space",
                 "annotations": {
                     "alarms": [
-                        "${aws_cloudwatch_metric_alarm.server_root_volume_disk_space[count.index].arn}"
+                        "${aws_cloudwatch_metric_alarm.server_root_volume_disk_space.arn}"
                     ]
                 },
                 "view": "timeSeries",
@@ -148,7 +142,7 @@ resource "aws_cloudwatch_dashboard" "riju" {
                 "title": "Data volume disk space",
                 "annotations": {
                     "alarms": [
-                        "${aws_cloudwatch_metric_alarm.server_data_volume_disk_space[count.index].arn}"
+                        "${aws_cloudwatch_metric_alarm.server_data_volume_disk_space.arn}"
                     ]
                 },
                 "view": "timeSeries",
@@ -166,7 +160,7 @@ resource "aws_cloudwatch_dashboard" "riju" {
                 "title": "Memory",
                 "annotations": {
                     "alarms": [
-                        "${aws_cloudwatch_metric_alarm.server_memory[count.index].arn}"
+                        "${aws_cloudwatch_metric_alarm.server_memory.arn}"
                     ]
                 },
                 "view": "timeSeries",

tf/ec2.tf

@@ -0,0 +1,19 @@
+resource "aws_security_group" "deploy" {
+  name        = "riju-deploy"
+  description = "Security group for Riju CI"
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}

tf/iam.tf

@@ -15,6 +15,16 @@ resource "aws_iam_access_key" "deploy" {
 }
 
 data "aws_iam_policy_document" "deploy" {
+  statement {
+    actions = [
+      "ec2:RunInstances",
+    ]
+
+    resources = [
+      data.aws_ami.ci.arn,
+    ]
+  }
+
   statement {
     actions = [
       "ecr:GetAuthorizationToken",
@@ -114,6 +124,11 @@ resource "aws_iam_role_policy_attachment" "deploy" {
   policy_arn = aws_iam_policy.deploy.arn
 }
 
+resource "aws_iam_instance_profile" "deploy" {
+  name = "riju-deploy"
+  role = aws_iam_role.deploy.name
+}
+
 data "aws_iam_policy_document" "server" {
   statement {
     actions = [
@@ -194,55 +209,6 @@ resource "aws_iam_instance_profile" "server" {
   role = aws_iam_role.server.name
 }
 
-data "aws_iam_policy_document" "dev_server" {
-  statement {
-    actions = [
-      "*",
-    ]
-
-    resources = [
-      "*",
-    ]
-  }
-}
-
-resource "aws_iam_policy" "dev_server" {
-  name        = "riju-dev-server"
-  description = "Policy granting AWS administrative access from dev server"
-  policy      = data.aws_iam_policy_document.dev_server.json
-}
-
-data "aws_iam_policy_document" "dev_server_assume_role" {
-  statement {
-    actions = [
-      "sts:AssumeRole",
-    ]
-
-    principals {
-      type = "Service"
-      identifiers = [
-        "ec2.amazonaws.com",
-      ]
-    }
-  }
-}
-
-resource "aws_iam_role" "dev_server" {
-  name               = "riju-dev-server"
-  description        = "Role used by Riju dev server"
-  assume_role_policy = data.aws_iam_policy_document.dev_server_assume_role.json
-}
-
-resource "aws_iam_role_policy_attachment" "dev_server" {
-  role       = aws_iam_role.dev_server.name
-  policy_arn = aws_iam_policy.dev_server.arn
-}
-
-resource "aws_iam_instance_profile" "dev_server" {
-  name = "riju-dev-server"
-  role = aws_iam_role.dev_server.name
-}
-
 data "aws_iam_policy_document" "backup_assume_role" {
   statement {
     actions = [

tf/main.tf

@@ -23,8 +23,6 @@ locals {
     Terraform       = "Managed by Terraform"
     BillingCategory = "Riju"
   }
-
-  ami_available = lookup(data.external.env.result, "AMI_NAME", "") != "" ? true : false
 }
 
 provider "aws" {

tf/ssm.tf

@@ -0,0 +1,24 @@
+resource "aws_ssm_parameter" "ci_ami_id" {
+  name      = "riju-ci-ami-id"
+  type      = "String"
+  value     = data.aws_ami.ci.id
+  data_type = "aws:ec2:image"
+}
+
+resource "aws_ssm_parameter" "docker_repo" {
+  name  = "riju-docker-repo-host"
+  type  = "String"
+  value = aws_ecr_repository.riju.repository_url
+}
+
+resource "aws_ssm_parameter" "public_docker_repo" {
+  name  = "riju-public-docker-repo-host"
+  type  = "String"
+  value = aws_ecrpublic_repository.riju.repository_uri
+}
+
+resource "aws_ssm_parameter" "s3_bucket" {
+  name  = "riju-s3-bucket-name"
+  type  = "String"
+  value = aws_s3_bucket.riju.bucket
+}

tools/ci-ec2.bash

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+remote_url="$(git remote get-url origin | sed -E 's|git@github\.com:|https://github.com/|')"
+commit="$(git rev-parse HEAD)"
+
+echo >&2 "[ci-run.bash] Fetching build parameters from SSM."
+
+parameters="riju-ci-ami-id riju-docker-repo-host riju-public-docker-repo-host riju-s3-bucket-name"
+resp="$(aws ssm get-parameters --names ${parameters})"
+
+read -r ami < <(jq '.Parameters[] | select(.Name == "riju-ci-ami-id").Value' -r <<< "${resp}")
+read -r docker_repo < <(jq '.Parameters[] | select(.Name == "riju-docker-repo-host").Value' -r <<< "${resp}")
+read -r public_docker_repo < <(jq '.Parameters[] | select(.Name == "riju-public-docker-repo-host").Value' -r <<< "${resp}")
+read -r s3_bucket < <(jq '.Parameters[] | select(.Name == "riju-s3-bucket-name").Value' -r <<< "${resp}")
+
+echo >&2 "[ci-run.bash] Launching EC2 instance for CI job."
+
+ebs_config="DeviceName=/dev/sdh,Ebs={DeleteOnTermination=true,VolumeSize=128,VolumeType=gp3}"
+instance_tags="ResourceType=instance,Tags=[{Key=Name,Value=Riju CI},{Key=BillingCategory,Value=Riju},{Key=BillingSubcategory,Value=Riju:EC2:CI}]"
+ebs_tags="ResourceType=volume,Tags=[{Key=Name,Value=Riju CI},{Key=BillingCategory,Value=Riju},{Key=BillingSubcategory,Value=Riju:EBS:CI}]"
+
+resp="$(aws ec2 run-instances                                      \
+            --image-id "${ami}"                                    \
+            --instance-type t3.2xlarge                             \
+            --security-groups riju-deploy                          \
+            --iam-instance-profile Name=riju-deploy                \
+            --instance-initiated-shutdown-behavior terminate       \
+            --user-data file://tools/ci-user-data.bash             \
+            --tag-specifications "${instance_tags}" "${ebs_tags}"  \
+            --block-device-mappings "${ebs_config}")"
+
+instance_id="$(jq '.Instances[].InstanceId' -r <<< "${resp}")"
+
+echo >&2 "[ci-run.bash] Waiting for instance ${instance_id} to become ready."
+
+success=
+for i in $(seq 1 15); do
+    sleep 2
+    resp="$(aws ec2 describe-instance-status --instance-id "${instance_id}")"
+    status="$(jq '.InstanceStatuses[].InstanceState.Name' -r <<< "${resp}")"
+    status="${status:-unknown}"
+    case "${status}" in
+        pending|unknown)             ;;
+        running) success=yes; break  ;;
+        *      ) exit 1              ;;
+    esac
+done
+
+if [[ -z "${success}}" ]]; then
+    exit 124
+fi
+
+echo >&2 "[ci-run.bash] Waiting for SSH to come online."
+
+success=
+for i in $(seq 1 15); do
+    if (yes || true) | timeout 5 mssh "ubuntu@${instance_id}" true 2>/dev/null; then
+        success=yes
+        break
+    elif (( $# == 124 )); then
+        exit 1
+    fi
+    sleep 2
+done
+
+if [[ -z "${success}}" ]]; then
+    exit 124
+fi
+
+echo >&2 "[ci-run.bash] Running CI remotely using EC2 Instance Connect."
+
+mssh "ubuntu@${instance_id}" bash <<EOF
+
+set -euo pipefail
+
+sudo riju-init-volume
+sudo chown ubuntu:ubuntu /mnt/riju
+
+pushd /mnt/riju
+
+mkdir -p src
+pushd src
+
+git clone ${remote_url} riju -b ${commit}
+pushd riju
+
+./tools/ci-bootstrap.bash
+
+popd
+popd
+popd
+
+EOF
+
+echo >&2 "[ci-run.bash] CI completed."

tools/ci-user-data.bash

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ -z "${NOHUP:-}" ]]; then
+    NOHUP=1 nohup "$0" "$@" &
+fi
+
+while true; do
+    sleep 60
+    # https://unix.stackexchange.com/a/92579
+    if ! sudo netstat -tnpa | grep 'ESTABLISHED.*sshd'; then
+        sudo shutdown -h now
+    fi
+done

tools/packer-build-ci.bash

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+packer build ci.pkr.hcl

tools/packer-build-web.bash

@@ -2,6 +2,10 @@
 
 set -euo pipefail
 
+: ${ADMIN_PASSWORD}
+: ${S3_BUCKET}
+: ${SUPERVISOR_ACCESS_TOKEN}
+
 export AWS_REGION="${AWS_REGION:-$(aws configure get region)}"
 
 if [[ -z "${AWS_REGION}" ]]; then
@@ -9,5 +13,4 @@ if [[ -z "${AWS_REGION}" ]]; then
     exit 1
 fi
 
-cd packer
+packer build web.pkr.hcl
-packer build config.pkr.hcl