Many misc updates

Makefile

@@ -22,7 +22,7 @@ endif
 # Get rid of 'Entering directory' / 'Leaving directory' messages.
 MAKE_QUIETLY := MAKELEVEL= make
 
-.PHONY: all $(MAKECMDGOALS)
+.PHONY: all $(MAKECMDGOALS) frontend system supervisor
 
  all: help
 

langs/abc.yaml

@@ -20,6 +20,7 @@ install:
 repl: |
   abc
 input: |
+  DELAY: 1
   WRITE 123 * 234
 
 main: "main.abc"

lib/yaml.js

@@ -2,6 +2,7 @@ import { promises as fs } from "fs";
 import path from "path";
 
 import { validate as validateJSONSchema } from "jsonschema";
+import _ from "lodash";
 import YAML from "yaml";
 
 // The build scripts in the language configs assume a specific build
@@ -120,7 +121,7 @@ export async function readSharedDepConfig(lang) {
 
 // Given a language config JSON, return a list of the Riju shared
 // dependency names, or an empty list if none are configured for this
-// language.
+// language. The return value is sorted.
 export async function getSharedDepsForLangConfig(langConfig) {
-  return (langConfig.install && langConfig.install.riju) || [];
+  return [...(langConfig.install && langConfig.install.riju) || []].sort();
 }

packer/provision.bash

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 : ${ADMIN_PASSWORD}
+: ${AWS_REGION}
 : ${S3_BUCKET}
 : ${SUPERVISOR_ACCESS_TOKEN}
 
@@ -23,9 +24,9 @@ ubuntu_name="$(lsb_release -cs)"
 sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null <<EOF
 deb [arch=amd64] https://download.docker.com/linux/ubuntu ${ubuntu_name} stable
 EOF
-}
+
 sudo -E apt-get update
-sudo -E apt-get install -y certbot docker-ce docker-ce-cli containerd.io unzip whois
+sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io unzip whois
 
 wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip
 unzip -q awscli.zip
@@ -38,9 +39,9 @@ sudo mv /tmp/riju.service /etc/systemd/system/
 sudo sed -Ei 's/^#?PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config
 sudo sed -Ei 's/^#?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config
 sudo sed -Ei 's/^#?PermitEmptyPasswords .*/PermitEmptyPasswords no/' /etc/ssh/sshd_config
-sudo sed -Ei "s/\$AWS_REGION/${AWS_REGION}/" /etc/systemd/system/riju.service
-sudo sed -Ei "s/\$S3_BUCKET/${S3_BUCKET}/" /etc/systemd/system/riju.service
-sudo sed -Ei "s/\$SUPERVISOR_ACCESS_TOKEN/${SUPERVISOR_ACCESS_TOKEN}/" /etc/systemd/system/riju.service
+sudo sed -Ei "s/\\\$AWS_REGION/${AWS_REGION}/" /etc/systemd/system/riju.service
+sudo sed -Ei "s/\\\$S3_BUCKET/${S3_BUCKET}/" /etc/systemd/system/riju.service
+sudo sed -Ei "s/\\\$SUPERVISOR_ACCESS_TOKEN/${SUPERVISOR_ACCESS_TOKEN}/" /etc/systemd/system/riju.service
 
 sudo passwd -l root
 sudo useradd admin -g admin -G sudo -s /usr/bin/bash -p "$(echo "${ADMIN_PASSWORD}" | mkpasswd -s)" -m

supervisor/src/main.go

@@ -312,9 +312,7 @@ func main() {
 		log.Fatalln(err)
 	}
 
-	stsClient := sts.New(sts.Options{
-		Region: awsCfg.Region,
-	})
+	stsClient := sts.NewFromConfig(awsCfg)
 	ident, err := stsClient.GetCallerIdentity(context.Background(), &sts.GetCallerIdentityInput{})
 	if err != nil {
 		log.Fatalln(err)

tf/asg.tf

@@ -51,7 +51,11 @@ resource "aws_launch_template" "server" {
   name = "riju-server"
   image_id = data.aws_ami.server[0].id
   instance_type = "t3.small"
+
   security_group_names = [aws_security_group.server.name]
+  iam_instance_profile {
+    name = aws_iam_instance_profile.server.name
+  }
 
   update_default_version = true
 
@@ -83,8 +87,8 @@ resource "aws_autoscaling_group" "server" {
   availability_zones = [
     for subnet in data.aws_subnet.default : subnet.availability_zone
   ]
-  desired_capacity = 1
-  min_size = 1
+  desired_capacity = 0
+  min_size = 0
   max_size = 3
 
   launch_template {

tf/iam.tf

@@ -30,7 +30,7 @@ data "aws_iam_policy_document" "deploy" {
 
 resource "aws_iam_policy" "deploy" {
   name        = "riju-deploy"
-  description = "Role used by CI to deploy Riju"
+  description = "Policy granting CI access to deploy Riju"
   policy      = data.aws_iam_policy_document.deploy.json
 }
 
@@ -39,34 +39,51 @@ resource "aws_iam_user_policy_attachment" "deploy" {
   policy_arn = aws_iam_policy.deploy.arn
 }
 
-data "aws_iam_policy_document" "riju" {
+data "aws_iam_policy_document" "server" {
   statement {
-    principals {
-      type        = "*"
-      identifiers = ["*"]
-    }
-
-    actions = [
-      "s3:ListBucket",
-    ]
-
-    resources = [
-      "arn:aws:s3:::${aws_s3_bucket.riju.bucket}",
-    ]
-  }
-
-  statement {
-    principals {
-      type        = "*"
-      identifiers = ["*"]
-    }
-
     actions = [
       "s3:GetObject",
     ]
 
     resources = [
-      "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*",
+      "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/config.json",
     ]
   }
 }
+
+resource "aws_iam_policy" "server" {
+  name = "riju-server"
+  description = "Policy granting supervisor process on Riju server ability to download from S3"
+  policy = data.aws_iam_policy_document.server.json
+}
+
+data "aws_iam_policy_document" "server_assume_role" {
+  statement {
+    actions = [
+      "sts:AssumeRole",
+    ]
+
+    principals {
+      type = "Service"
+      identifiers = [
+        "ec2.amazonaws.com",
+      ]
+    }
+  }
+}
+
+resource "aws_iam_role" "server" {
+  name = "riju-server"
+  description = "Role used by supervisor process on Riju server"
+  assume_role_policy = data.aws_iam_policy_document.server_assume_role.json
+}
+
+resource "aws_iam_role_policy_attachment" "server" {
+  role = aws_iam_role.server.name
+  policy_arn = aws_iam_policy.server.arn
+}
+
+resource "aws_iam_instance_profile" "server" {
+  name = "riju-server"
+  role = aws_iam_role.server.name
+}

tf/s3.tf

@@ -11,7 +11,39 @@ resource "aws_s3_bucket_public_access_block" "riju" {
   restrict_public_buckets = true
 }
 
+data "aws_iam_policy_document" "s3" {
+  statement {
+    principals {
+      type        = "*"
+      identifiers = ["*"]
+    }
+
+    actions = [
+      "s3:ListBucket",
+    ]
+
+    resources = [
+      "arn:aws:s3:::${aws_s3_bucket.riju.bucket}",
+    ]
+  }
+
+  statement {
+    principals {
+      type        = "*"
+      identifiers = ["*"]
+    }
+
+    actions = [
+      "s3:GetObject",
+    ]
+
+    resources = [
+      "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*",
+    ]
+  }
+}
+
 resource "aws_s3_bucket_policy" "riju" {
   bucket = aws_s3_bucket.riju.id
-  policy = data.aws_iam_policy_document.riju.json
+  policy = data.aws_iam_policy_document.s3.json
 }

tools/build-lang-image.js

@@ -27,10 +27,15 @@ async function main() {
   program.option("--debug", "interactive debugging");
   program.parse(process.argv);
   const { lang, debug } = program.opts();
+  const sharedDeps = await getSharedDepsForLangConfig(await readLangConfig(lang));
   const installContents = await fs.readFile(
     `build/lang/${lang}/install.bash`,
     "utf-8"
   );
+  const sharedInstallContents = await Promise.all(sharedDeps.map(
+    async (name) => fs.readFile(`build/shared/${name}/install.bash`),
+  ));
+  const allInstallContents = [].concat.apply([installContents], sharedInstallContents);
   const hash = await hashDockerfile(
     "lang",
     {
@@ -41,13 +46,15 @@ async function main() {
         langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`),
         sharedHashes: (
           await Promise.all(
-            (await getSharedDepsForLangConfig(await readLangConfig(lang))).map(
+            sharedDeps.map(
               async (name) =>
                 await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`)
             )
           )
         ).sort(),
-        installHash: crypto.createHash("sha1").update(installContents).digest("hex"),
+        installHash: allInstallContents.map(
+          (c) => crypto.createHash("sha1").update(c).digest("hex"),
+        ).join(""),
       },
     }
   );

tools/depgraph.js

@@ -124,12 +124,18 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) {
           `build/lang/${isLangImage.lang}/install.bash`,
           "utf-8"
         );
+        const sharedInstallContents = await Promise.all(isLangImage.sharedDeps.map(
+          async (name) => fs.readFile(`build/shared/${name}/install.bash`),
+        ));
+        const allInstallContents = [].concat.apply([installContents], sharedInstallContents);
         salt = {
           langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`],
           sharedHashes: isLangImage.sharedDeps.map(
             (name) => dependencyHashes[`deb:shared-${name}`]
           ),
-          installHash: crypto.createHash("sha1").update(installContents).digest("hex"),
+          installHash: allInstallContents.map(
+            (c) => crypto.createHash("sha1").update(c).digest("hex"),
+          ).join(""),
         };
       }
       return await hashDockerfile(name, dependentDockerHashes, { salt });

tools/packer-build.bash

@@ -4,7 +4,7 @@ set -euo pipefail
 
 export AWS_REGION="${AWS_REGION:-$(aws configure get region)}"
 
-if [[ -n "${AWS_REGION}" ]]; then
+if [[ -z "${AWS_REGION}" ]]; then
     echo >&2 "no default AWS region specified, and AWS_REGION unset"
     exit 1
 fi