MirrorMirror
/
riju
mirror of https://github.com/radian-software/riju.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add code formatters for Riju itself

This commit is contained in:
Radon Rosborough 2021-07-17 00:50:42 -07:00
parent 85d0a42371
commit 0bec96b88b
30 changed files with 345 additions and 366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.clang-format Normal file
View File

@ -0,0 +1 @@
BreakBeforeBraces: Linux

19
Makefile
View File

@ -255,6 +255,25 @@ deploy-latest: # Upload deployment config to S3 and update ASG instances
deploy: deploy-config deploy-latest # Shorthand for deploy-config followed by deploy-latest deploy: deploy-config deploy-latest # Shorthand for deploy-config followed by deploy-latest
### Code formatting
fmt-c: # Format C code
git ls-files | grep -E '\.c$$' | xargs clang-format -i
fmt-go: # Format Go code
git ls-files | grep -E '\.go$' | xargs gofmt -l -w
fmt-python: # Format Python code
git ls-files | grep -E '\.py$$' | xargs black -q
fmt-terraform: # Format Terraform code
terraform fmt "$(PWD)/tf"
fmt-web: # Format CSS, JSON, and YAML code
git ls-files | grep -E '\.(|css|c?js|json|ya?ml)$$' | grep -Ev '^(langs|shared)/' | xargs prettier --write --loglevel=warn
fmt: fmt-c fmt-go fmt-python fmt-terraform fmt-web # Format all code
### Infrastructure ### Infrastructure
packer: supervisor # Build and publish a new AMI packer: supervisor # Build and publish a new AMI

22
backend/api.js
View File

@ -95,7 +95,7 @@ export class Session {
event: "serviceLog", event: "serviceLog",
service: "container", service: "container",
output: line + "\n", output: line + "\n",
}) });
} }
} }
}); });
@ -293,16 +293,8 @@ export class Session {
runCode = async (code) => { runCode = async (code) => {
try { try {
const { const { name, repl, main, suffix, createEmpty, compile, run, template } =
name, this.config;
repl,
main,
suffix,
createEmpty,
compile,
run,
template,
} = this.config;
if (this.term) { if (this.term) {
try { try {
process.kill(this.term.pty.pid); process.kill(this.term.pty.pid);
@ -433,9 +425,11 @@ export class Session {
}; };
ensure = async (cmd) => { ensure = async (cmd) => {
const code = (await this.run(this.privilegedExec(cmd), { const code = (
check: false, await this.run(this.privilegedExec(cmd), {
})).code; check: false,
})
).code;
this.send({ event: "ensured", code }); this.send({ event: "ensured", code });
}; };

5
backend/langs.js
View File

@ -44,4 +44,7 @@ async function updateLangsFromDisk() {
export const langsPromise = updateLangsFromDisk().then(() => langs); export const langsPromise = updateLangsFromDisk().then(() => langs);
export const langWatcher = fsOrig.watch("langs", debounce(updateLangsFromDisk, 200)); export const langWatcher = fsOrig.watch(
"langs",
debounce(updateLangsFromDisk, 200)
);

2
backend/sandbox.js
View File

@ -57,7 +57,7 @@ async function main() {
{ uuid }, { uuid },
bash( bash(
`env L='${lang}' LANG_CONFIG=${quote( `env L='${lang}' LANG_CONFIG=${quote(
JSON.stringify(langConfig), JSON.stringify(langConfig)
)} bash --rcfile <(cat <<< ${quote(sandboxScript)})` )} bash --rcfile <(cat <<< ${quote(sandboxScript)})`
) )
); );

93
backend/test-runner.js
View File

@ -318,32 +318,8 @@ class Test {
dynamicRegistration: true, dynamicRegistration: true,
symbolKind: { symbolKind: {
valueSet: [ valueSet: [
1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
2, 18, 19, 20, 21, 22, 23, 24, 25, 26,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
], ],
}, },
}, },
@ -376,31 +352,8 @@ class Test {
}, },
completionItemKind: { completionItemKind: {
valueSet: [ valueSet: [
1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
2, 18, 19, 20, 21, 22, 23, 24, 25,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
], ],
}, },
}, },
@ -423,32 +376,8 @@ class Test {
dynamicRegistration: true, dynamicRegistration: true,
symbolKind: { symbolKind: {
valueSet: [ valueSet: [
1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
2, 18, 19, 20, 21, 22, 23, 24, 25, 26,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
], ],
}, },
hierarchicalDocumentSymbolSupport: true, hierarchicalDocumentSymbolSupport: true,
@ -638,9 +567,11 @@ async function writeLog(lang, type, result, log) {
} }
async function getImageHash(tag) { async function getImageHash(tag) {
const output = (await run(["docker", "inspect", `riju:${tag}`], console.error, { const output = (
suppressOutput: true, await run(["docker", "inspect", `riju:${tag}`], console.error, {
})).output; suppressOutput: true,
})
).output;
return JSON.parse(output)[0].Config.Labels["riju.image-hash"]; return JSON.parse(output)[0].Config.Labels["riju.image-hash"];
} }
@ -777,7 +708,7 @@ async function main() {
await fs.mkdir(`build/test-hashes/lang`, { recursive: true }); await fs.mkdir(`build/test-hashes/lang`, { recursive: true });
await fs.writeFile( await fs.writeFile(
`build/test-hashes/lang/${lang}`, `build/test-hashes/lang/${lang}`,
await getTestHash(lang, runtimeHash, langHashes[lang]), await getTestHash(lang, runtimeHash, langHashes[lang])
); );
} }
process.exit(failed.size > 0 ? 1 : 0); process.exit(failed.size > 0 ? 1 : 0);

5
backend/util.js
View File

@ -6,12 +6,11 @@ import { v4 as getUUIDOrig } from "uuid";
function computeImageHashes() { function computeImageHashes() {
let deployConfig = process.env.RIJU_DEPLOY_CONFIG; let deployConfig = process.env.RIJU_DEPLOY_CONFIG;
if (!deployConfig) if (!deployConfig) return {};
return {};
deployConfig = JSON.parse(deployConfig); deployConfig = JSON.parse(deployConfig);
const imageHashes = {}; const imageHashes = {};
for (const [lang, tag] of Object.entries(deployConfig.langImageTags)) { for (const [lang, tag] of Object.entries(deployConfig.langImageTags)) {
const prefix = `lang-${lang}-` const prefix = `lang-${lang}-`;
if (!tag.startsWith(prefix)) { if (!tag.startsWith(prefix)) {
throw new Error(`malformed tag ${tag}`); throw new Error(`malformed tag ${tag}`);
} }

5
docker/admin/install.bash
View File

@ -32,7 +32,9 @@ packages="
apt-file apt-file
bind9-dnsutils bind9-dnsutils
black
clang clang
clang-format
dctrl-tools dctrl-tools
docker-ce-cli docker-ce-cli
g++ g++
@ -47,6 +49,7 @@ man
moreutils moreutils
nodejs nodejs
packer packer
prettier
psmisc psmisc
python3-pip python3-pip
pwgen pwgen
@ -68,6 +71,8 @@ apt-get install -y $(sed 's/#.*//' <<< "${packages}")
pip3 install ec2instanceconnectcli pip3 install ec2instanceconnectcli
npm install -g prettier
wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip
unzip -q awscli.zip unzip -q awscli.zip
./aws/install ./aws/install

4
lib/hash-test.js
View File

@ -82,7 +82,9 @@ export async function getTestHash(lang, runtimeImageHash, langImageHash) {
return crypto return crypto
.createHash("sha1") .createHash("sha1")
.update( .update(
`${await testRunnerHash},${await getTestConfigHash(lang)},${runtimeImageHash},${langImageHash}` `${await testRunnerHash},${await getTestConfigHash(
lang
)},${runtimeImageHash},${langImageHash}`
) )
.digest("hex"); .digest("hex");
} }

1
lib/jsonschema.yaml
View File

@ -6,7 +6,6 @@ type: object
additionalProperties: false additionalProperties: false
required: [id, name, main, template, run] required: [id, name, main, template, run]
properties: properties:
id: id:
title: "Canonical language ID" title: "Canonical language ID"
description: | description: |

5
lib/yaml.js
View File

@ -27,7 +27,8 @@ const jsonSchemaPromise = readJSONSchemaFromDisk();
export async function getLangs() { export async function getLangs() {
return (await fs.readdir("langs")) return (await fs.readdir("langs"))
.filter((lang) => lang.endsWith(".yaml")) .filter((lang) => lang.endsWith(".yaml"))
.map((lang) => path.parse(lang).name).sort(); .map((lang) => path.parse(lang).name)
.sort();
} }
// Return a list of the IDs of all the configured shared dependencies. // Return a list of the IDs of all the configured shared dependencies.
@ -123,5 +124,5 @@ export async function readSharedDepConfig(lang) {
// dependency names, or an empty list if none are configured for this // dependency names, or an empty list if none are configured for this
// language. The return value is sorted. // language. The return value is sorted.
export async function getSharedDepsForLangConfig(langConfig) { export async function getSharedDepsForLangConfig(langConfig) {
return [...(langConfig.install && langConfig.install.riju) || []].sort(); return [...((langConfig.install && langConfig.install.riju) || [])].sort();
} }

16
packer/cloudwatch.json
View File

@ -16,24 +16,16 @@
], ],
"metrics_collected": { "metrics_collected": {
"cpu": { "cpu": {
"measurement": [ "measurement": ["usage_active"],
"usage_active"
],
"metrics_collection_interval": 60 "metrics_collection_interval": 60
}, },
"disk": { "disk": {
"measurement": [ "measurement": ["used_percent"],
"used_percent"
],
"metrics_collection_interval": 60, "metrics_collection_interval": 60,
"resources": [ "resources": ["*"]
"*"
]
}, },
"mem": { "mem": {
"measurement": [ "measurement": ["mem_used_percent"],
"mem_used_percent"
],
"metrics_collection_interval": 60 "metrics_collection_interval": 60
} }
} }

54
supervisor/src/main.go
View File

@ -38,13 +38,13 @@ const blueName = "riju-app-blue"
const greenName = "riju-app-green" const greenName = "riju-app-green"
type deploymentConfig struct { type deploymentConfig struct {
AppImageTag string `json:"appImageTag"` AppImageTag string `json:"appImageTag"`
LangImageTags map[string]string `json:"langImageTags"` LangImageTags map[string]string `json:"langImageTags"`
} }
type supervisorConfig struct { type supervisorConfig struct {
AccessToken string `env:"SUPERVISOR_ACCESS_TOKEN,notEmpty"` AccessToken string `env:"SUPERVISOR_ACCESS_TOKEN,notEmpty"`
S3Bucket string `env:"S3_BUCKET,notEmpty"` S3Bucket string `env:"S3_BUCKET,notEmpty"`
} }
type reloadJob struct { type reloadJob struct {
@ -56,22 +56,22 @@ type reloadJob struct {
type supervisor struct { type supervisor struct {
config supervisorConfig config supervisorConfig
blueProxyHandler http.Handler blueProxyHandler http.Handler
greenProxyHandler http.Handler greenProxyHandler http.Handler
isGreen bool // blue-green deployment isGreen bool // blue-green deployment
deployConfigHash string deployConfigHash string
awsAccountNumber string awsAccountNumber string
awsRegion string awsRegion string
s3 *s3.Client s3 *s3.Client
ecr *ecr.Client ecr *ecr.Client
reloadLock sync.Mutex reloadLock sync.Mutex
reloadInProgress bool reloadInProgress bool
reloadNeeded bool reloadNeeded bool
reloadUUID string reloadUUID string
reloadNextUUID string reloadNextUUID string
reloadJobs map[string]*reloadJob reloadJobs map[string]*reloadJob
} }
func (sv *supervisor) status(status string) { func (sv *supervisor) status(status string) {
@ -113,7 +113,7 @@ func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, "401 malformed Authorization header", http.StatusUnauthorized) http.Error(w, "401 malformed Authorization header", http.StatusUnauthorized)
return return
} }
if authHeader != "Bearer " + sv.config.AccessToken { if authHeader != "Bearer "+sv.config.AccessToken {
http.Error(w, "401 wrong access token", http.StatusUnauthorized) http.Error(w, "401 wrong access token", http.StatusUnauthorized)
return return
} }
@ -149,11 +149,11 @@ func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, "404 no such job", http.StatusNotFound) http.Error(w, "404 no such job", http.StatusNotFound)
} }
} else if job.active { } else if job.active {
fmt.Fprintln(w, "active: " + job.status) fmt.Fprintln(w, "active: "+job.status)
} else if job.failed { } else if job.failed {
fmt.Fprintln(w, "failed: " + job.status) fmt.Fprintln(w, "failed: "+job.status)
} else { } else {
fmt.Fprintln(w, "succeeded: " + job.status) fmt.Fprintln(w, "succeeded: "+job.status)
} }
sv.reloadLock.Unlock() sv.reloadLock.Unlock()
return return
@ -256,7 +256,7 @@ func (sv *supervisor) reload() error {
buf := s3manager.NewWriteAtBuffer([]byte{}) buf := s3manager.NewWriteAtBuffer([]byte{})
if _, err := dl.Download(context.Background(), buf, &s3.GetObjectInput{ if _, err := dl.Download(context.Background(), buf, &s3.GetObjectInput{
Bucket: &sv.config.S3Bucket, Bucket: &sv.config.S3Bucket,
Key: aws.String("config.json"), Key: aws.String("config.json"),
}); err != nil { }); err != nil {
return err return err
} }
@ -575,16 +575,16 @@ func main() {
} }
sv := &supervisor{ sv := &supervisor{
config: supervisorCfg, config: supervisorCfg,
blueProxyHandler: httputil.NewSingleHostReverseProxy(blueUrl), blueProxyHandler: httputil.NewSingleHostReverseProxy(blueUrl),
greenProxyHandler: httputil.NewSingleHostReverseProxy(greenUrl), greenProxyHandler: httputil.NewSingleHostReverseProxy(greenUrl),
isGreen: isGreen, isGreen: isGreen,
deployConfigHash: deployCfgHash, deployConfigHash: deployCfgHash,
s3: s3.NewFromConfig(awsCfg), s3: s3.NewFromConfig(awsCfg),
ecr: ecr.NewFromConfig(awsCfg), ecr: ecr.NewFromConfig(awsCfg),
awsRegion: awsCfg.Region, awsRegion: awsCfg.Region,
awsAccountNumber: *ident.Account, awsAccountNumber: *ident.Account,
reloadJobs: map[string]*reloadJob{}, reloadJobs: map[string]*reloadJob{},
} }
go sv.scheduleReload() go sv.scheduleReload()
log.Println("listening on http://0.0.0.0:80") log.Println("listening on http://0.0.0.0:80")

51
system/res/docker-exec.py
View File

@ -6,6 +6,7 @@ import subprocess
import sys import sys
import uuid import uuid
class Parser(argparse.ArgumentParser): class Parser(argparse.ArgumentParser):
def format_help(self): def format_help(self):
return """ return """
@ -19,6 +20,7 @@ Options:
-u, --user string Username or UID (format: <name|uid>:[<group|gid>]) -u, --user string Username or UID (format: <name|uid>:[<group|gid>])
""" """
parser = Parser() parser = Parser()
parser.add_argument("-i", "--interactive", action="store_true") parser.add_argument("-i", "--interactive", action="store_true")
parser.add_argument("-t", "--tty", action="store_true") parser.add_argument("-t", "--tty", action="store_true")
@ -34,20 +36,23 @@ pidfile = pidfiles + "/" + str(uuid.uuid4()).replace("-", "")
# We have to use 'kill -9' here, otherwise runuser intercepts the # We have to use 'kill -9' here, otherwise runuser intercepts the
# signal and takes its sweet time cleaning up. # signal and takes its sweet time cleaning up.
def cleanup(*ignored_args): def cleanup(*ignored_args):
subprocess.run([ subprocess.run(
"docker", [
"exec", "docker",
args.container, "exec",
"bash", args.container,
"-c", "bash",
f""" "-c",
f"""
set -euo pipefail set -euo pipefail
if [[ -f '{pidfile}' ]]; then if [[ -f '{pidfile}' ]]; then
kill -9 -$(< '{pidfile}') 2>/dev/null || true kill -9 -$(< '{pidfile}') 2>/dev/null || true
rm -f '{pidfile}' rm -f '{pidfile}'
fi fi
""" """,
]) ]
)
signal.signal(signal.SIGINT, cleanup) signal.signal(signal.SIGINT, cleanup)
signal.signal(signal.SIGTERM, cleanup) signal.signal(signal.SIGTERM, cleanup)
@ -64,21 +69,25 @@ runuser_args = []
if args.user: if args.user:
runuser_args = ["runuser", "-u", args.user, "--"] runuser_args = ["runuser", "-u", args.user, "--"]
sys.exit(subprocess.run([ sys.exit(
"docker", subprocess.run(
"exec", [
*exec_args, "docker",
args.container, "exec",
"bash", *exec_args,
"-c", args.container,
f""" "bash",
"-c",
f"""
set -euo pipefail set -euo pipefail
umask 077 umask 077
mkdir -p '{pidfiles}' mkdir -p '{pidfiles}'
echo "$$" > '{pidfile}' echo "$$" > '{pidfile}'
exec "$@" exec "$@"
""", """,
"--", "--",
*runuser_args, *runuser_args,
*args.arg, *args.arg,
]).returncode) ]
).returncode
)

104
system/src/riju-system-privileged.c
View File

@ -1,6 +1,6 @@
#define _GNU_SOURCE #define _GNU_SOURCE
#include <fcntl.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h>
#include <grp.h> #include <grp.h>
#include <signal.h> #include <signal.h>
#include <stdbool.h> #include <stdbool.h>
@ -13,7 +13,7 @@
#include <time.h> #include <time.h>
#include <unistd.h> #include <unistd.h>
void __attribute__ ((noreturn)) die(char *msg) void __attribute__((noreturn)) die(char *msg)
{ {
fprintf(stderr, "%s\n", msg); fprintf(stderr, "%s\n", msg);
exit(1); exit(1);
@ -37,7 +37,8 @@ char *parseUUID(char *uuid)
return uuid; return uuid;
} }
char *parseLang(char *lang) { char *parseLang(char *lang)
{
size_t len = strnlen(lang, 65); size_t len = strnlen(lang, 65);
if (len == 0 || len > 64) if (len == 0 || len > 64)
die("illegal language name"); die("illegal language name");
@ -63,9 +64,8 @@ void wait_alarm(int signum)
void session(char *uuid, char *lang, char *imageHash) void session(char *uuid, char *lang, char *imageHash)
{ {
char *image, *container, *hostname, *volume, *fifo; char *image, *container, *hostname, *volume, *fifo;
if ((imageHash != NULL ? if ((imageHash != NULL ? asprintf(&image, "riju:lang-%s-%s", lang, imageHash)
asprintf(&image, "riju:lang-%s-%s", lang, imageHash) : : asprintf(&image, "riju:lang-%s", lang)) < 0)
asprintf(&image, "riju:lang-%s", lang)) < 0)
die("asprintf failed"); die("asprintf failed");
if (asprintf(&container, "riju-session-%s", uuid) < 0) if (asprintf(&container, "riju-session-%s", uuid) < 0)
die("asprintf failed"); die("asprintf failed");
@ -88,37 +88,60 @@ void session(char *uuid, char *lang, char *imageHash)
die("fork failed"); die("fork failed");
else if (pid == 0) { else if (pid == 0) {
char *argv[] = { char *argv[] = {
"docker", "docker",
"run", "run",
"--rm", "--rm",
"-v", volume, "-v",
"-e", "HOME=/home/riju", volume,
"-e", hostname, "-e",
"-e", "LANG=C.UTF-8", "HOME=/home/riju",
"-e", "LC_ALL=C.UTF-8", "-e",
"-e", "LOGNAME=riju", hostname,
"-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin", "-e",
"-e", "PWD=/home/riju/src", "LANG=C.UTF-8",
"-e", "SHELL=/usr/bin/bash", "-e",
"-e", "TERM=xterm-256color", "LC_ALL=C.UTF-8",
"-e", "TMPDIR=/tmp", "-e",
"-e", "USER=riju", "LOGNAME=riju",
"-e", "USERNAME=riju", "-e",
"--user", "root", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin",
"--hostname", lang, "-e",
"--name", container, "PWD=/home/riju/src",
"--cpus", "1", "-e",
"--memory", "1g", "SHELL=/usr/bin/bash",
"--memory-swap", "3g", "-e",
"--pids-limit", "512", "TERM=xterm-256color",
image, "bash", "-c", "-e",
"cat /var/run/riju/sentinel/fifo | ( sleep 10; while read -t2; do :; done; pkill -g0 )", "TMPDIR=/tmp",
NULL, "-e",
"USER=riju",
"-e",
"USERNAME=riju",
"--user",
"root",
"--hostname",
lang,
"--name",
container,
"--cpus",
"1",
"--memory",
"1g",
"--memory-swap",
"3g",
"--pids-limit",
"512",
image,
"bash",
"-c",
"cat /var/run/riju/sentinel/fifo | ( sleep 10; while read -t2; do :; "
"done; pkill -g0 )",
NULL,
}; };
execvp(argv[0], argv); execvp(argv[0], argv);
die("execvp failed"); die("execvp failed");
} }
struct timespec ts_10ms; // 10ms struct timespec ts_10ms; // 10ms
ts_10ms.tv_sec = 0; ts_10ms.tv_sec = 0;
ts_10ms.tv_nsec = 1000 * 1000 * 10; ts_10ms.tv_nsec = 1000 * 1000 * 10;
signal(SIGALRM, wait_alarm); signal(SIGALRM, wait_alarm);
@ -143,7 +166,7 @@ void session(char *uuid, char *lang, char *imageHash)
if (pid < 0) if (pid < 0)
die("fork failed"); die("fork failed");
else if (pid == 0) { else if (pid == 0) {
struct timespec ts_1s; // 10ms struct timespec ts_1s; // 10ms
ts_1s.tv_sec = 1; ts_1s.tv_sec = 1;
ts_1s.tv_nsec = 0; ts_1s.tv_nsec = 0;
while (1) { while (1) {
@ -155,7 +178,7 @@ void session(char *uuid, char *lang, char *imageHash)
die("nanosleep failed"); die("nanosleep failed");
} }
} }
printf("riju: container ready\n"); // magic string printf("riju: container ready\n"); // magic string
if (waitpid(pid, NULL, 0) <= 0) if (waitpid(pid, NULL, 0) <= 0)
die("waitpid failed"); die("waitpid failed");
if (close(fd) < 0) if (close(fd) < 0)
@ -168,11 +191,12 @@ void exec(char *uuid, int argc, char **cmdline, bool pty)
if (asprintf(&container, "riju-session-%s", uuid) < 0) if (asprintf(&container, "riju-session-%s", uuid) < 0)
die("asprintf failed"); die("asprintf failed");
char *argvPrefix[] = { char *argvPrefix[] = {
"./system/res/docker-exec.py", "./system/res/docker-exec.py",
"--user", "riju", "--user",
pty ? "-it" : "-i", "riju",
container, pty ? "-it" : "-i",
"--", container,
"--",
}; };
char **argv = malloc(sizeof(argvPrefix) + (argc + 1) * sizeof(char *)); char **argv = malloc(sizeof(argvPrefix) + (argc + 1) * sizeof(char *));
if (argv == NULL) if (argv == NULL)

4
tf/acm.tf
View File

@ -1,7 +1,7 @@
resource "aws_acm_certificate" "riju" { resource "aws_acm_certificate" "riju" {
domain_name = "riju.codes" domain_name = "riju.codes"
subject_alternative_names = ["*.riju.codes"] subject_alternative_names = ["*.riju.codes"]
validation_method = "DNS" validation_method = "DNS"
tags = { tags = {
Name = "Riju server" Name = "Riju server"

32
tf/alb.tf
View File

@ -27,30 +27,30 @@ resource "aws_security_group" "alb" {
} }
resource "aws_lb" "server" { resource "aws_lb" "server" {
name = "riju-server" name = "riju-server"
security_groups = [aws_security_group.alb.id] security_groups = [aws_security_group.alb.id]
subnets = data.aws_subnet_ids.default.ids subnets = data.aws_subnet_ids.default.ids
idle_timeout = 3600 idle_timeout = 3600
} }
resource "aws_lb_target_group" "server" { resource "aws_lb_target_group" "server" {
name = "riju-server-http" name = "riju-server-http"
port = 80 port = 80
protocol = "HTTP" protocol = "HTTP"
vpc_id = data.aws_vpc.default.id vpc_id = data.aws_vpc.default.id
} }
resource "aws_lb_listener" "server_http" { resource "aws_lb_listener" "server_http" {
load_balancer_arn = aws_lb.server.arn load_balancer_arn = aws_lb.server.arn
port = "80" port = "80"
protocol = "HTTP" protocol = "HTTP"
default_action { default_action {
type = "redirect" type = "redirect"
redirect { redirect {
port = "443" port = "443"
protocol = "HTTPS" protocol = "HTTPS"
status_code = "HTTP_301" status_code = "HTTP_301"
} }
} }
@ -58,13 +58,13 @@ resource "aws_lb_listener" "server_http" {
resource "aws_lb_listener" "server_https" { resource "aws_lb_listener" "server_https" {
load_balancer_arn = aws_lb.server.arn load_balancer_arn = aws_lb.server.arn
port = "443" port = "443"
protocol = "HTTPS" protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08" ssl_policy = "ELBSecurityPolicy-2016-08"
certificate_arn = aws_acm_certificate.riju.arn certificate_arn = aws_acm_certificate.riju.arn
default_action { default_action {
type = "forward" type = "forward"
target_group_arn = aws_lb_target_group.server.arn target_group_arn = aws_lb_target_group.server.arn
} }
} }
@ -73,5 +73,5 @@ resource "aws_autoscaling_attachment" "server" {
count = local.ami_available ? 1 : 0 count = local.ami_available ? 1 : 0
autoscaling_group_name = aws_autoscaling_group.server[count.index].name autoscaling_group_name = aws_autoscaling_group.server[count.index].name
alb_target_group_arn = aws_lb_target_group.server.arn alb_target_group_arn = aws_lb_target_group.server.arn
} }

6
tf/ami.tf
View File

@ -15,17 +15,17 @@ data "aws_ami" "ubuntu" {
owners = ["099720109477"] owners = ["099720109477"]
filter { filter {
name = "name" name = "name"
values = ["ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*"] values = ["ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*"]
} }
filter { filter {
name = "root-device-type" name = "root-device-type"
values = ["ebs"] values = ["ebs"]
} }
filter { filter {
name = "virtualization-type" name = "virtualization-type"
values = ["hvm"] values = ["hvm"]
} }

22
tf/asg.tf
View File

@ -37,8 +37,8 @@ resource "aws_security_group" "server" {
resource "aws_launch_template" "server" { resource "aws_launch_template" "server" {
count = local.ami_available ? 1 : 0 count = local.ami_available ? 1 : 0
name = "riju-server" name = "riju-server"
image_id = data.aws_ami.server[count.index].id image_id = data.aws_ami.server[count.index].id
instance_type = "t3.medium" instance_type = "t3.medium"
security_group_names = [aws_security_group.server.name] security_group_names = [aws_security_group.server.name]
@ -51,8 +51,8 @@ resource "aws_launch_template" "server" {
block_device_mappings { block_device_mappings {
device_name = "/dev/sdh" device_name = "/dev/sdh"
ebs { ebs {
volume_type = "gp3" volume_type = "gp3"
volume_size = 256 volume_size = 256
} }
} }
@ -77,8 +77,8 @@ resource "aws_autoscaling_group" "server" {
for subnet in data.aws_subnet.default : subnet.availability_zone for subnet in data.aws_subnet.default : subnet.availability_zone
] ]
desired_capacity = 1 desired_capacity = 1
min_size = 1 min_size = 1
max_size = 3 max_size = 3
launch_template { launch_template {
id = aws_launch_template.server[count.index].id id = aws_launch_template.server[count.index].id
@ -87,19 +87,19 @@ resource "aws_autoscaling_group" "server" {
tags = concat( tags = concat(
[ [
{ {
key = "Name" key = "Name"
value = "Riju server" value = "Riju server"
propagate_at_launch = false propagate_at_launch = false
} }
], ],
[ [
for key, value in local.tags : { for key, value in local.tags : {
key = key, key = key,
value = value, value = value,
propagate_at_launch = true, propagate_at_launch = true,
} }
], ],
) )
lifecycle { lifecycle {
ignore_changes = [target_group_arns] ignore_changes = [target_group_arns]

8
tf/backup.tf
View File

@ -6,9 +6,9 @@ resource "aws_backup_plan" "riju" {
name = "riju" name = "riju"
rule { rule {
rule_name = "riju" rule_name = "riju"
target_vault_name = aws_backup_vault.riju.name target_vault_name = aws_backup_vault.riju.name
schedule = "cron(0 5 ? * * *)" schedule = "cron(0 5 ? * * *)"
lifecycle { lifecycle {
delete_after = 3 delete_after = 3
@ -24,8 +24,8 @@ resource "aws_backup_selection" "riju" {
count = local.ssh_key_available ? 1 : 0 count = local.ssh_key_available ? 1 : 0
iam_role_arn = aws_iam_role.backup.arn iam_role_arn = aws_iam_role.backup.arn
name = "riju" name = "riju"
plan_id = aws_backup_plan.riju.id plan_id = aws_backup_plan.riju.id
resources = [ resources = [
aws_instance.dev_server[count.index].arn, aws_instance.dev_server[count.index].arn,

92
tf/cloudwatch.tf
View File

@ -1,17 +1,17 @@
resource "aws_cloudwatch_metric_alarm" "server_cpu" { resource "aws_cloudwatch_metric_alarm" "server_cpu" {
count = local.ami_available ? 1 : 0 count = local.ami_available ? 1 : 0
alarm_name = "riju-server-cpu-high" alarm_name = "riju-server-cpu-high"
comparison_operator = "GreaterThanOrEqualToThreshold" comparison_operator = "GreaterThanOrEqualToThreshold"
evaluation_periods = "30" evaluation_periods = "30"
metric_name = "cpu_usage_active" metric_name = "cpu_usage_active"
namespace = "CWAgent" namespace = "CWAgent"
period = "60" period = "60"
statistic = "Maximum" statistic = "Maximum"
threshold = "90" threshold = "90"
alarm_description = "CPU usage on Riju server is above 90% for 30 minutes" alarm_description = "CPU usage on Riju server is above 90% for 30 minutes"
ok_actions = [aws_sns_topic.riju.arn] ok_actions = [aws_sns_topic.riju.arn]
alarm_actions = [aws_sns_topic.riju.arn] alarm_actions = [aws_sns_topic.riju.arn]
insufficient_data_actions = [aws_sns_topic.riju.arn] insufficient_data_actions = [aws_sns_topic.riju.arn]
dimensions = { dimensions = {
AutoScalingGroupName = aws_autoscaling_group.server[count.index].name AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
@ -21,17 +21,17 @@ resource "aws_cloudwatch_metric_alarm" "server_cpu" {
resource "aws_cloudwatch_metric_alarm" "server_memory" { resource "aws_cloudwatch_metric_alarm" "server_memory" {
count = local.ami_available ? 1 : 0 count = local.ami_available ? 1 : 0
alarm_name = "riju-server-memory-high" alarm_name = "riju-server-memory-high"
comparison_operator = "GreaterThanOrEqualToThreshold" comparison_operator = "GreaterThanOrEqualToThreshold"
evaluation_periods = "30" evaluation_periods = "30"
metric_name = "mem_used_percent" metric_name = "mem_used_percent"
namespace = "CWAgent" namespace = "CWAgent"
period = "60" period = "60"
statistic = "Maximum" statistic = "Maximum"
threshold = "80" threshold = "80"
alarm_description = "Memory usage on Riju server is above 80% for 30 minutes" alarm_description = "Memory usage on Riju server is above 80% for 30 minutes"
ok_actions = [aws_sns_topic.riju.arn] ok_actions = [aws_sns_topic.riju.arn]
alarm_actions = [aws_sns_topic.riju.arn] alarm_actions = [aws_sns_topic.riju.arn]
insufficient_data_actions = [aws_sns_topic.riju.arn] insufficient_data_actions = [aws_sns_topic.riju.arn]
dimensions = { dimensions = {
AutoScalingGroupName = aws_autoscaling_group.server[count.index].name AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
@ -41,42 +41,42 @@ resource "aws_cloudwatch_metric_alarm" "server_memory" {
resource "aws_cloudwatch_metric_alarm" "server_data_volume_disk_space" { resource "aws_cloudwatch_metric_alarm" "server_data_volume_disk_space" {
count = local.ami_available ? 1 : 0 count = local.ami_available ? 1 : 0
alarm_name = "riju-server-data-volume-disk-usage-high" alarm_name = "riju-server-data-volume-disk-usage-high"
comparison_operator = "GreaterThanOrEqualToThreshold" comparison_operator = "GreaterThanOrEqualToThreshold"
evaluation_periods = "30" evaluation_periods = "30"
metric_name = "disk_used_percent" metric_name = "disk_used_percent"
namespace = "CWAgent" namespace = "CWAgent"
period = "60" period = "60"
statistic = "Maximum" statistic = "Maximum"
threshold = "90" threshold = "90"
alarm_description = "Disk space usage for data volume on Riju server is above 90% for 30 minutes" alarm_description = "Disk space usage for data volume on Riju server is above 90% for 30 minutes"
ok_actions = [aws_sns_topic.riju.arn] ok_actions = [aws_sns_topic.riju.arn]
alarm_actions = [aws_sns_topic.riju.arn] alarm_actions = [aws_sns_topic.riju.arn]
insufficient_data_actions = [aws_sns_topic.riju.arn] insufficient_data_actions = [aws_sns_topic.riju.arn]
dimensions = { dimensions = {
AutoScalingGroupName = aws_autoscaling_group.server[count.index].name AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
path = "/mnt/riju/data" path = "/mnt/riju/data"
} }
} }
resource "aws_cloudwatch_metric_alarm" "server_root_volume_disk_space" { resource "aws_cloudwatch_metric_alarm" "server_root_volume_disk_space" {
count = local.ami_available ? 1 : 0 count = local.ami_available ? 1 : 0
alarm_name = "riju-server-root-volume-disk-usage-high" alarm_name = "riju-server-root-volume-disk-usage-high"
comparison_operator = "GreaterThanOrEqualToThreshold" comparison_operator = "GreaterThanOrEqualToThreshold"
evaluation_periods = "30" evaluation_periods = "30"
metric_name = "disk_used_percent" metric_name = "disk_used_percent"
namespace = "CWAgent" namespace = "CWAgent"
period = "60" period = "60"
statistic = "Maximum" statistic = "Maximum"
threshold = "90" threshold = "90"
alarm_description = "Disk space usage for root volume on Riju server is above 90% for 30 minutes" alarm_description = "Disk space usage for root volume on Riju server is above 90% for 30 minutes"
ok_actions = [aws_sns_topic.riju.arn] ok_actions = [aws_sns_topic.riju.arn]
alarm_actions = [aws_sns_topic.riju.arn] alarm_actions = [aws_sns_topic.riju.arn]
insufficient_data_actions = [aws_sns_topic.riju.arn] insufficient_data_actions = [aws_sns_topic.riju.arn]
dimensions = { dimensions = {
AutoScalingGroupName = aws_autoscaling_group.server[count.index].name AutoScalingGroupName = aws_autoscaling_group.server[count.index].name
path = "/" path = "/"
} }
} }

10
tf/ec2.tf
View File

@ -31,14 +31,14 @@ resource "aws_security_group" "dev_server" {
resource "aws_instance" "dev_server" { resource "aws_instance" "dev_server" {
count = local.ssh_key_available ? 1 : 0 count = local.ssh_key_available ? 1 : 0
ami = data.aws_ami.ubuntu[count.index].id ami = data.aws_ami.ubuntu[count.index].id
instance_type = "t3.2xlarge" instance_type = "t3.2xlarge"
ebs_optimized = true ebs_optimized = true
security_groups = [aws_security_group.dev_server[count.index].name] security_groups = [aws_security_group.dev_server[count.index].name]
iam_instance_profile = aws_iam_instance_profile.dev_server.name iam_instance_profile = aws_iam_instance_profile.dev_server.name
key_name = data.external.env.result.SSH_KEY_NAME key_name = data.external.env.result.SSH_KEY_NAME
root_block_device { root_block_device {
volume_size = 256 volume_size = 256
@ -56,7 +56,7 @@ resource "aws_instance" "dev_server" {
lifecycle { lifecycle {
ignore_changes = [ ignore_changes = [
ami, ami,
security_groups, # legacy security_groups, # legacy
] ]
} }
} }
@ -69,7 +69,7 @@ resource "aws_eip" "dev_server" {
} }
resource "aws_eip_association" "dev_server" { resource "aws_eip_association" "dev_server" {
count = local.ssh_key_available ? 1 : 0 count = local.ssh_key_available ? 1 : 0
instance_id = aws_instance.dev_server[count.index].id instance_id = aws_instance.dev_server[count.index].id
allocation_id = aws_eip.dev_server[count.index].id allocation_id = aws_eip.dev_server[count.index].id
} }

2
tf/ecr.tf
View File

@ -4,6 +4,6 @@ resource "aws_ecr_repository" "riju" {
} }
resource "aws_ecrpublic_repository" "riju" { resource "aws_ecrpublic_repository" "riju" {
provider = aws.us_east_1 provider = aws.us_east_1
repository_name = "riju" repository_name = "riju"
} }

38
tf/iam.tf
View File

@ -104,13 +104,13 @@ resource "aws_iam_user_policy_attachment" "deploy" {
} }
resource "aws_iam_role" "deploy" { resource "aws_iam_role" "deploy" {
name = "riju-deploy" name = "riju-deploy"
description = "Role used by CI and deployment" description = "Role used by CI and deployment"
assume_role_policy = data.aws_iam_policy_document.deploy_assume_role.json assume_role_policy = data.aws_iam_policy_document.deploy_assume_role.json
} }
resource "aws_iam_role_policy_attachment" "deploy" { resource "aws_iam_role_policy_attachment" "deploy" {
role = aws_iam_role.deploy.name role = aws_iam_role.deploy.name
policy_arn = aws_iam_policy.deploy.arn policy_arn = aws_iam_policy.deploy.arn
} }
@ -148,9 +148,9 @@ data "aws_iam_policy_document" "server" {
} }
resource "aws_iam_policy" "server" { resource "aws_iam_policy" "server" {
name = "riju-server" name = "riju-server"
description = "Policy granting supervisor process on Riju server ability to download from S3" description = "Policy granting supervisor process on Riju server ability to download from S3"
policy = data.aws_iam_policy_document.server.json policy = data.aws_iam_policy_document.server.json
} }
data "aws_iam_policy_document" "server_assume_role" { data "aws_iam_policy_document" "server_assume_role" {
@ -169,23 +169,23 @@ data "aws_iam_policy_document" "server_assume_role" {
} }
resource "aws_iam_role" "server" { resource "aws_iam_role" "server" {
name = "riju-server" name = "riju-server"
description = "Role used by supervisor process on Riju server" description = "Role used by supervisor process on Riju server"
assume_role_policy = data.aws_iam_policy_document.server_assume_role.json assume_role_policy = data.aws_iam_policy_document.server_assume_role.json
} }
resource "aws_iam_role_policy_attachment" "server" { resource "aws_iam_role_policy_attachment" "server" {
role = aws_iam_role.server.name role = aws_iam_role.server.name
policy_arn = aws_iam_policy.server.arn policy_arn = aws_iam_policy.server.arn
} }
resource "aws_iam_role_policy_attachment" "server_cloudwatch" { resource "aws_iam_role_policy_attachment" "server_cloudwatch" {
role = aws_iam_role.server.name role = aws_iam_role.server.name
policy_arn = data.aws_iam_policy.cloudwatch.arn policy_arn = data.aws_iam_policy.cloudwatch.arn
} }
resource "aws_iam_role_policy_attachment" "server_ssm" { resource "aws_iam_role_policy_attachment" "server_ssm" {
role = aws_iam_role.server.name role = aws_iam_role.server.name
policy_arn = data.aws_iam_policy.ssm.arn policy_arn = data.aws_iam_policy.ssm.arn
} }
@ -207,9 +207,9 @@ data "aws_iam_policy_document" "dev_server" {
} }
resource "aws_iam_policy" "dev_server" { resource "aws_iam_policy" "dev_server" {
name = "riju-dev-server" name = "riju-dev-server"
description = "Policy granting AWS administrative access from dev server" description = "Policy granting AWS administrative access from dev server"
policy = data.aws_iam_policy_document.dev_server.json policy = data.aws_iam_policy_document.dev_server.json
} }
data "aws_iam_policy_document" "dev_server_assume_role" { data "aws_iam_policy_document" "dev_server_assume_role" {
@ -228,13 +228,13 @@ data "aws_iam_policy_document" "dev_server_assume_role" {
} }
resource "aws_iam_role" "dev_server" { resource "aws_iam_role" "dev_server" {
name = "riju-dev-server" name = "riju-dev-server"
description = "Role used by Riju dev server" description = "Role used by Riju dev server"
assume_role_policy = data.aws_iam_policy_document.dev_server_assume_role.json assume_role_policy = data.aws_iam_policy_document.dev_server_assume_role.json
} }
resource "aws_iam_role_policy_attachment" "dev_server" { resource "aws_iam_role_policy_attachment" "dev_server" {
role = aws_iam_role.dev_server.name role = aws_iam_role.dev_server.name
policy_arn = aws_iam_policy.dev_server.arn policy_arn = aws_iam_policy.dev_server.arn
} }
@ -259,8 +259,8 @@ data "aws_iam_policy_document" "backup_assume_role" {
} }
resource "aws_iam_role" "backup" { resource "aws_iam_role" "backup" {
name = "riju-backup" name = "riju-backup"
description = "Role used by AWS Backup for Riju" description = "Role used by AWS Backup for Riju"
assume_role_policy = data.aws_iam_policy_document.backup_assume_role.json assume_role_policy = data.aws_iam_policy_document.backup_assume_role.json
} }
@ -273,11 +273,11 @@ data "aws_iam_policy" "backup_restores" {
} }
resource "aws_iam_role_policy_attachment" "backup" { resource "aws_iam_role_policy_attachment" "backup" {
role = aws_iam_role.backup.name role = aws_iam_role.backup.name
policy_arn = data.aws_iam_policy.backup.arn policy_arn = data.aws_iam_policy.backup.arn
} }
resource "aws_iam_role_policy_attachment" "backup_restores" { resource "aws_iam_role_policy_attachment" "backup_restores" {
role = aws_iam_role.backup.name role = aws_iam_role.backup.name
policy_arn = data.aws_iam_policy.backup_restores.arn policy_arn = data.aws_iam_policy.backup_restores.arn
} }

10
tf/main.tf
View File

@ -1,6 +1,6 @@
terraform { terraform {
backend "s3" { backend "s3" {
key = "state" key = "state"
} }
required_providers { required_providers {
aws = { aws = {
@ -8,7 +8,7 @@ terraform {
version = "~> 3.45" version = "~> 3.45"
} }
null = { null = {
source = "hashicorp/null" source = "hashicorp/null"
version = "~> 3.1" version = "~> 3.1"
} }
} }
@ -24,7 +24,7 @@ locals {
BillingCategory = "Riju" BillingCategory = "Riju"
} }
ami_available = lookup(data.external.env.result, "AMI_NAME", "") != "" ? true : false ami_available = lookup(data.external.env.result, "AMI_NAME", "") != "" ? true : false
ssh_key_available = lookup(data.external.env.result, "SSH_KEY_NAME", "") != "" ? true : false ssh_key_available = lookup(data.external.env.result, "SSH_KEY_NAME", "") != "" ? true : false
} }
@ -35,7 +35,7 @@ provider "aws" {
} }
provider "aws" { provider "aws" {
alias = "us_east_1" alias = "us_east_1"
region = "us-east-1" region = "us-east-1"
default_tags { default_tags {
tags = local.tags tags = local.tags
@ -56,5 +56,5 @@ data "aws_subnet_ids" "default" {
data "aws_subnet" "default" { data "aws_subnet" "default" {
for_each = data.aws_subnet_ids.default.ids for_each = data.aws_subnet_ids.default.ids
id = each.value id = each.value
} }

23
tools/build-lang-image.js
View File

@ -27,15 +27,22 @@ async function main() {
program.option("--debug", "interactive debugging"); program.option("--debug", "interactive debugging");
program.parse(process.argv); program.parse(process.argv);
const { lang, debug } = program.opts(); const { lang, debug } = program.opts();
const sharedDeps = await getSharedDepsForLangConfig(await readLangConfig(lang)); const sharedDeps = await getSharedDepsForLangConfig(
await readLangConfig(lang)
);
const installContents = await fs.readFile( const installContents = await fs.readFile(
`build/lang/${lang}/install.bash`, `build/lang/${lang}/install.bash`,
"utf-8" "utf-8"
); );
const sharedInstallContents = await Promise.all(sharedDeps.map( const sharedInstallContents = await Promise.all(
async (name) => fs.readFile(`build/shared/${name}/install.bash`), sharedDeps.map(async (name) =>
)); fs.readFile(`build/shared/${name}/install.bash`)
const allInstallContents = [].concat.apply([installContents], sharedInstallContents); )
);
const allInstallContents = [].concat.apply(
[installContents],
sharedInstallContents
);
const hash = await hashDockerfile( const hash = await hashDockerfile(
"lang", "lang",
{ {
@ -52,9 +59,9 @@ async function main() {
) )
) )
).sort(), ).sort(),
installHash: allInstallContents.map( installHash: allInstallContents
(c) => crypto.createHash("sha1").update(c).digest("hex"), .map((c) => crypto.createHash("sha1").update(c).digest("hex"))
).join(""), .join(""),
}, },
} }
); );

36
tools/depgraph.js
View File

@ -124,18 +124,23 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) {
`build/lang/${isLangImage.lang}/install.bash`, `build/lang/${isLangImage.lang}/install.bash`,
"utf-8" "utf-8"
); );
const sharedInstallContents = await Promise.all(isLangImage.sharedDeps.map( const sharedInstallContents = await Promise.all(
async (name) => fs.readFile(`build/shared/${name}/install.bash`), isLangImage.sharedDeps.map(async (name) =>
)); fs.readFile(`build/shared/${name}/install.bash`)
const allInstallContents = [].concat.apply([installContents], sharedInstallContents); )
);
const allInstallContents = [].concat.apply(
[installContents],
sharedInstallContents
);
salt = { salt = {
langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`], langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`],
sharedHashes: isLangImage.sharedDeps.map( sharedHashes: isLangImage.sharedDeps.map(
(name) => dependencyHashes[`deb:shared-${name}`] (name) => dependencyHashes[`deb:shared-${name}`]
), ),
installHash: allInstallContents.map( installHash: allInstallContents
(c) => crypto.createHash("sha1").update(c).digest("hex"), .map((c) => crypto.createHash("sha1").update(c).digest("hex"))
).join(""), .join(""),
}; };
} }
return await hashDockerfile(name, dependentDockerHashes, { salt }); return await hashDockerfile(name, dependentDockerHashes, { salt });
@ -231,7 +236,7 @@ async function getLanguageTestArtifact({ lang }) {
return await getTestHash( return await getTestHash(
lang, lang,
dependencyHashes[`image:runtime`], dependencyHashes[`image:runtime`],
dependencyHashes[`image:lang-${lang}`], dependencyHashes[`image:lang-${lang}`]
); );
}, },
buildLocally: async () => { buildLocally: async () => {
@ -248,10 +253,10 @@ async function getLanguageTestArtifact({ lang }) {
const hash = (await fs.readFile(hashPath, "utf-8")).trim(); const hash = (await fs.readFile(hashPath, "utf-8")).trim();
const S3_BUCKET = getS3Bucket(); const S3_BUCKET = getS3Bucket();
await runCommand( await runCommand(
`aws s3 rm --recursive s3://${S3_BUCKET}/test-hashes/lang/${lang}`, `aws s3 rm --recursive s3://${S3_BUCKET}/test-hashes/lang/${lang}`
); );
await runCommand( await runCommand(
`aws s3 cp ${hashPath} s3://${S3_BUCKET}/test-hashes/lang/${lang}/${hash}`, `aws s3 cp ${hashPath} s3://${S3_BUCKET}/test-hashes/lang/${lang}/${hash}`
); );
}, },
}; };
@ -658,15 +663,8 @@ async function main() {
program.option("--publish", "publish artifacts to remote registries"); program.option("--publish", "publish artifacts to remote registries");
program.option("--yes", "execute plan without confirmation"); program.option("--yes", "execute plan without confirmation");
program.parse(process.argv); program.parse(process.argv);
const { const { list, manual, holdManual, all, localOnly, publish, yes } =
list, program.opts();
manual,
holdManual,
all,
localOnly,
publish,
yes,
} = program.opts();
const depgraph = await getDepGraph(); const depgraph = await getDepGraph();
if (list) { if (list) {
for (const { name } of depgraph.artifacts) { for (const { name } of depgraph.artifacts) {

23
tools/generate-build-script.js
View File

@ -41,16 +41,8 @@ function makeLangScript(langConfig, isShared) {
deb, deb,
} = install; } = install;
if (prepare) { if (prepare) {
const { const { preface, cert, aptKey, aptRepo, apt, npm, opam, manual } =
preface, prepare;
cert,
aptKey,
aptRepo,
apt,
npm,
opam,
manual,
} = prepare;
if (preface) { if (preface) {
prefaceParts.push(preface); prefaceParts.push(preface);
} }
@ -450,9 +442,11 @@ export async function generateBuildScript({ lang, type }) {
const buildScriptPath = `build/${type}/${lang}/build.bash`; const buildScriptPath = `build/${type}/${lang}/build.bash`;
const installScriptPath = `build/${type}/${lang}/install.bash`; const installScriptPath = `build/${type}/${lang}/install.bash`;
await Promise.all([ await Promise.all([
fs.writeFile(buildScriptPath, buildScript + "\n") fs
.writeFile(buildScriptPath, buildScript + "\n")
.then(() => fs.chmod(buildScriptPath, 0o755)), .then(() => fs.chmod(buildScriptPath, 0o755)),
fs.writeFile(installScriptPath, installScript + "\n") fs
.writeFile(installScriptPath, installScript + "\n")
.then(() => fs.chmod(installScriptPath, 0o755)), .then(() => fs.chmod(installScriptPath, 0o755)),
]); ]);
} }
@ -462,10 +456,7 @@ async function main() {
const program = new Command(); const program = new Command();
program program
.requiredOption("--lang <id>", "language ID") .requiredOption("--lang <id>", "language ID")
.requiredOption( .requiredOption("--type <value>", "package category (lang or shared)");
"--type <value>",
"package category (lang or shared)"
);
program.parse(process.argv); program.parse(process.argv);
await generateBuildScript(program.opts()); await generateBuildScript(program.opts());
process.exit(0); process.exit(0);

12
tools/generate-deploy-config.js
View File

@ -15,15 +15,16 @@ async function getDeployConfig() {
langs.map(async (lang) => [ langs.map(async (lang) => [
lang, lang,
`lang-${lang}-` + `lang-${lang}-` +
(await getLocalImageLabel(`riju:lang-${lang}`, "riju.image-hash")), (await getLocalImageLabel(`riju:lang-${lang}`, "riju.image-hash")),
]) ])
) )
); );
const appImageTag = `app-` + await getLocalImageLabel(`riju:app`, "riju.image-hash"); const appImageTag =
`app-` + (await getLocalImageLabel(`riju:app`, "riju.image-hash"));
return { return {
appImageTag, appImageTag,
langImageTags, langImageTags,
} };
} }
// Parse command-line arguments, run main functionality, and exit. // Parse command-line arguments, run main functionality, and exit.
@ -31,7 +32,10 @@ async function main() {
const program = new Command(); const program = new Command();
program.parse(process.argv); program.parse(process.argv);
await fs.mkdir("build", { recursive: true }); await fs.mkdir("build", { recursive: true });
await fs.writeFile("build/config.json", JSON.stringify(await getDeployConfig(), null, 2) + "\n"); await fs.writeFile(
"build/config.json",
JSON.stringify(await getDeployConfig(), null, 2) + "\n"
);
console.log("wrote build/config.json"); console.log("wrote build/config.json");
process.exit(0); process.exit(0);
} }

6
tools/hash-dockerfile.js
View File

@ -139,9 +139,9 @@ async function encodeDockerfile(name, dependentHashes, opts) {
} }
step.context = await Promise.all( step.context = await Promise.all(
sources.map(async (source) => sources.map(async (source) =>
(await listFiles(source)).filter( (
(entry) => !ignore.ignores(entry.path) await listFiles(source)
) ).filter((entry) => !ignore.ignores(entry.path))
) )
); );
break; break;