Add code formatters for Riju itself

This commit is contained in:
Radon Rosborough 2021-07-17 00:50:42 -07:00
parent 85d0a42371
commit 0bec96b88b
30 changed files with 345 additions and 366 deletions

1
.clang-format Normal file
View File

@ -0,0 +1 @@
BreakBeforeBraces: Linux

View File

@ -255,6 +255,25 @@ deploy-latest: # Upload deployment config to S3 and update ASG instances
deploy: deploy-config deploy-latest # Shorthand for deploy-config followed by deploy-latest
### Code formatting
fmt-c: # Format C code
git ls-files | grep -E '\.c$$' | xargs clang-format -i
fmt-go: # Format Go code
git ls-files | grep -E '\.go$' | xargs gofmt -l -w
fmt-python: # Format Python code
git ls-files | grep -E '\.py$$' | xargs black -q
fmt-terraform: # Format Terraform code
terraform fmt "$(PWD)/tf"
fmt-web: # Format CSS, JSON, and YAML code
git ls-files | grep -E '\.(|css|c?js|json|ya?ml)$$' | grep -Ev '^(langs|shared)/' | xargs prettier --write --loglevel=warn
fmt: fmt-c fmt-go fmt-python fmt-terraform fmt-web # Format all code
### Infrastructure
packer: supervisor # Build and publish a new AMI

View File

@ -95,7 +95,7 @@ export class Session {
event: "serviceLog",
service: "container",
output: line + "\n",
})
});
}
}
});
@ -293,16 +293,8 @@ export class Session {
runCode = async (code) => {
try {
const {
name,
repl,
main,
suffix,
createEmpty,
compile,
run,
template,
} = this.config;
const { name, repl, main, suffix, createEmpty, compile, run, template } =
this.config;
if (this.term) {
try {
process.kill(this.term.pty.pid);
@ -433,9 +425,11 @@ export class Session {
};
ensure = async (cmd) => {
const code = (await this.run(this.privilegedExec(cmd), {
const code = (
await this.run(this.privilegedExec(cmd), {
check: false,
})).code;
})
).code;
this.send({ event: "ensured", code });
};

View File

@ -44,4 +44,7 @@ async function updateLangsFromDisk() {
export const langsPromise = updateLangsFromDisk().then(() => langs);
export const langWatcher = fsOrig.watch("langs", debounce(updateLangsFromDisk, 200));
export const langWatcher = fsOrig.watch(
"langs",
debounce(updateLangsFromDisk, 200)
);

View File

@ -57,7 +57,7 @@ async function main() {
{ uuid },
bash(
`env L='${lang}' LANG_CONFIG=${quote(
JSON.stringify(langConfig),
JSON.stringify(langConfig)
)} bash --rcfile <(cat <<< ${quote(sandboxScript)})`
)
);

View File

@ -318,32 +318,8 @@ class Test {
dynamicRegistration: true,
symbolKind: {
valueSet: [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26,
],
},
},
@ -376,31 +352,8 @@ class Test {
},
completionItemKind: {
valueSet: [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25,
],
},
},
@ -423,32 +376,8 @@ class Test {
dynamicRegistration: true,
symbolKind: {
valueSet: [
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26,
],
},
hierarchicalDocumentSymbolSupport: true,
@ -638,9 +567,11 @@ async function writeLog(lang, type, result, log) {
}
async function getImageHash(tag) {
const output = (await run(["docker", "inspect", `riju:${tag}`], console.error, {
const output = (
await run(["docker", "inspect", `riju:${tag}`], console.error, {
suppressOutput: true,
})).output;
})
).output;
return JSON.parse(output)[0].Config.Labels["riju.image-hash"];
}
@ -777,7 +708,7 @@ async function main() {
await fs.mkdir(`build/test-hashes/lang`, { recursive: true });
await fs.writeFile(
`build/test-hashes/lang/${lang}`,
await getTestHash(lang, runtimeHash, langHashes[lang]),
await getTestHash(lang, runtimeHash, langHashes[lang])
);
}
process.exit(failed.size > 0 ? 1 : 0);

View File

@ -6,12 +6,11 @@ import { v4 as getUUIDOrig } from "uuid";
function computeImageHashes() {
let deployConfig = process.env.RIJU_DEPLOY_CONFIG;
if (!deployConfig)
return {};
if (!deployConfig) return {};
deployConfig = JSON.parse(deployConfig);
const imageHashes = {};
for (const [lang, tag] of Object.entries(deployConfig.langImageTags)) {
const prefix = `lang-${lang}-`
const prefix = `lang-${lang}-`;
if (!tag.startsWith(prefix)) {
throw new Error(`malformed tag ${tag}`);
}

View File

@ -32,7 +32,9 @@ packages="
apt-file
bind9-dnsutils
black
clang
clang-format
dctrl-tools
docker-ce-cli
g++
@ -47,6 +49,7 @@ man
moreutils
nodejs
packer
prettier
psmisc
python3-pip
pwgen
@ -68,6 +71,8 @@ apt-get install -y $(sed 's/#.*//' <<< "${packages}")
pip3 install ec2instanceconnectcli
npm install -g prettier
wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip
unzip -q awscli.zip
./aws/install

View File

@ -82,7 +82,9 @@ export async function getTestHash(lang, runtimeImageHash, langImageHash) {
return crypto
.createHash("sha1")
.update(
`${await testRunnerHash},${await getTestConfigHash(lang)},${runtimeImageHash},${langImageHash}`
`${await testRunnerHash},${await getTestConfigHash(
lang
)},${runtimeImageHash},${langImageHash}`
)
.digest("hex");
}

View File

@ -6,7 +6,6 @@ type: object
additionalProperties: false
required: [id, name, main, template, run]
properties:
id:
title: "Canonical language ID"
description: |

View File

@ -27,7 +27,8 @@ const jsonSchemaPromise = readJSONSchemaFromDisk();
export async function getLangs() {
return (await fs.readdir("langs"))
.filter((lang) => lang.endsWith(".yaml"))
.map((lang) => path.parse(lang).name).sort();
.map((lang) => path.parse(lang).name)
.sort();
}
// Return a list of the IDs of all the configured shared dependencies.
@ -123,5 +124,5 @@ export async function readSharedDepConfig(lang) {
// dependency names, or an empty list if none are configured for this
// language. The return value is sorted.
export async function getSharedDepsForLangConfig(langConfig) {
return [...(langConfig.install && langConfig.install.riju) || []].sort();
return [...((langConfig.install && langConfig.install.riju) || [])].sort();
}

View File

@ -16,24 +16,16 @@
],
"metrics_collected": {
"cpu": {
"measurement": [
"usage_active"
],
"measurement": ["usage_active"],
"metrics_collection_interval": 60
},
"disk": {
"measurement": [
"used_percent"
],
"measurement": ["used_percent"],
"metrics_collection_interval": 60,
"resources": [
"*"
]
"resources": ["*"]
},
"mem": {
"measurement": [
"mem_used_percent"
],
"measurement": ["mem_used_percent"],
"metrics_collection_interval": 60
}
}

View File

@ -6,6 +6,7 @@ import subprocess
import sys
import uuid
class Parser(argparse.ArgumentParser):
def format_help(self):
return """
@ -19,6 +20,7 @@ Options:
-u, --user string Username or UID (format: <name|uid>:[<group|gid>])
"""
parser = Parser()
parser.add_argument("-i", "--interactive", action="store_true")
parser.add_argument("-t", "--tty", action="store_true")
@ -34,7 +36,8 @@ pidfile = pidfiles + "/" + str(uuid.uuid4()).replace("-", "")
# We have to use 'kill -9' here, otherwise runuser intercepts the
# signal and takes its sweet time cleaning up.
def cleanup(*ignored_args):
subprocess.run([
subprocess.run(
[
"docker",
"exec",
args.container,
@ -46,8 +49,10 @@ if [[ -f '{pidfile}' ]]; then
kill -9 -$(< '{pidfile}') 2>/dev/null || true
rm -f '{pidfile}'
fi
"""
])
""",
]
)
signal.signal(signal.SIGINT, cleanup)
signal.signal(signal.SIGTERM, cleanup)
@ -64,7 +69,9 @@ runuser_args = []
if args.user:
runuser_args = ["runuser", "-u", args.user, "--"]
sys.exit(subprocess.run([
sys.exit(
subprocess.run(
[
"docker",
"exec",
*exec_args,
@ -81,4 +88,6 @@ exec "$@"
"--",
*runuser_args,
*args.arg,
]).returncode)
]
).returncode
)

View File

@ -1,6 +1,6 @@
#define _GNU_SOURCE
#include <fcntl.h>
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <signal.h>
#include <stdbool.h>
@ -37,7 +37,8 @@ char *parseUUID(char *uuid)
return uuid;
}
char *parseLang(char *lang) {
char *parseLang(char *lang)
{
size_t len = strnlen(lang, 65);
if (len == 0 || len > 64)
die("illegal language name");
@ -63,9 +64,8 @@ void wait_alarm(int signum)
void session(char *uuid, char *lang, char *imageHash)
{
char *image, *container, *hostname, *volume, *fifo;
if ((imageHash != NULL ?
asprintf(&image, "riju:lang-%s-%s", lang, imageHash) :
asprintf(&image, "riju:lang-%s", lang)) < 0)
if ((imageHash != NULL ? asprintf(&image, "riju:lang-%s-%s", lang, imageHash)
: asprintf(&image, "riju:lang-%s", lang)) < 0)
die("asprintf failed");
if (asprintf(&container, "riju-session-%s", uuid) < 0)
die("asprintf failed");
@ -91,28 +91,51 @@ void session(char *uuid, char *lang, char *imageHash)
"docker",
"run",
"--rm",
"-v", volume,
"-e", "HOME=/home/riju",
"-e", hostname,
"-e", "LANG=C.UTF-8",
"-e", "LC_ALL=C.UTF-8",
"-e", "LOGNAME=riju",
"-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin",
"-e", "PWD=/home/riju/src",
"-e", "SHELL=/usr/bin/bash",
"-e", "TERM=xterm-256color",
"-e", "TMPDIR=/tmp",
"-e", "USER=riju",
"-e", "USERNAME=riju",
"--user", "root",
"--hostname", lang,
"--name", container,
"--cpus", "1",
"--memory", "1g",
"--memory-swap", "3g",
"--pids-limit", "512",
image, "bash", "-c",
"cat /var/run/riju/sentinel/fifo | ( sleep 10; while read -t2; do :; done; pkill -g0 )",
"-v",
volume,
"-e",
"HOME=/home/riju",
"-e",
hostname,
"-e",
"LANG=C.UTF-8",
"-e",
"LC_ALL=C.UTF-8",
"-e",
"LOGNAME=riju",
"-e",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin",
"-e",
"PWD=/home/riju/src",
"-e",
"SHELL=/usr/bin/bash",
"-e",
"TERM=xterm-256color",
"-e",
"TMPDIR=/tmp",
"-e",
"USER=riju",
"-e",
"USERNAME=riju",
"--user",
"root",
"--hostname",
lang,
"--name",
container,
"--cpus",
"1",
"--memory",
"1g",
"--memory-swap",
"3g",
"--pids-limit",
"512",
image,
"bash",
"-c",
"cat /var/run/riju/sentinel/fifo | ( sleep 10; while read -t2; do :; "
"done; pkill -g0 )",
NULL,
};
execvp(argv[0], argv);
@ -169,7 +192,8 @@ void exec(char *uuid, int argc, char **cmdline, bool pty)
die("asprintf failed");
char *argvPrefix[] = {
"./system/res/docker-exec.py",
"--user", "riju",
"--user",
"riju",
pty ? "-it" : "-i",
container,
"--",

View File

@ -27,15 +27,22 @@ async function main() {
program.option("--debug", "interactive debugging");
program.parse(process.argv);
const { lang, debug } = program.opts();
const sharedDeps = await getSharedDepsForLangConfig(await readLangConfig(lang));
const sharedDeps = await getSharedDepsForLangConfig(
await readLangConfig(lang)
);
const installContents = await fs.readFile(
`build/lang/${lang}/install.bash`,
"utf-8"
);
const sharedInstallContents = await Promise.all(sharedDeps.map(
async (name) => fs.readFile(`build/shared/${name}/install.bash`),
));
const allInstallContents = [].concat.apply([installContents], sharedInstallContents);
const sharedInstallContents = await Promise.all(
sharedDeps.map(async (name) =>
fs.readFile(`build/shared/${name}/install.bash`)
)
);
const allInstallContents = [].concat.apply(
[installContents],
sharedInstallContents
);
const hash = await hashDockerfile(
"lang",
{
@ -52,9 +59,9 @@ async function main() {
)
)
).sort(),
installHash: allInstallContents.map(
(c) => crypto.createHash("sha1").update(c).digest("hex"),
).join(""),
installHash: allInstallContents
.map((c) => crypto.createHash("sha1").update(c).digest("hex"))
.join(""),
},
}
);

View File

@ -124,18 +124,23 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) {
`build/lang/${isLangImage.lang}/install.bash`,
"utf-8"
);
const sharedInstallContents = await Promise.all(isLangImage.sharedDeps.map(
async (name) => fs.readFile(`build/shared/${name}/install.bash`),
));
const allInstallContents = [].concat.apply([installContents], sharedInstallContents);
const sharedInstallContents = await Promise.all(
isLangImage.sharedDeps.map(async (name) =>
fs.readFile(`build/shared/${name}/install.bash`)
)
);
const allInstallContents = [].concat.apply(
[installContents],
sharedInstallContents
);
salt = {
langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`],
sharedHashes: isLangImage.sharedDeps.map(
(name) => dependencyHashes[`deb:shared-${name}`]
),
installHash: allInstallContents.map(
(c) => crypto.createHash("sha1").update(c).digest("hex"),
).join(""),
installHash: allInstallContents
.map((c) => crypto.createHash("sha1").update(c).digest("hex"))
.join(""),
};
}
return await hashDockerfile(name, dependentDockerHashes, { salt });
@ -231,7 +236,7 @@ async function getLanguageTestArtifact({ lang }) {
return await getTestHash(
lang,
dependencyHashes[`image:runtime`],
dependencyHashes[`image:lang-${lang}`],
dependencyHashes[`image:lang-${lang}`]
);
},
buildLocally: async () => {
@ -248,10 +253,10 @@ async function getLanguageTestArtifact({ lang }) {
const hash = (await fs.readFile(hashPath, "utf-8")).trim();
const S3_BUCKET = getS3Bucket();
await runCommand(
`aws s3 rm --recursive s3://${S3_BUCKET}/test-hashes/lang/${lang}`,
`aws s3 rm --recursive s3://${S3_BUCKET}/test-hashes/lang/${lang}`
);
await runCommand(
`aws s3 cp ${hashPath} s3://${S3_BUCKET}/test-hashes/lang/${lang}/${hash}`,
`aws s3 cp ${hashPath} s3://${S3_BUCKET}/test-hashes/lang/${lang}/${hash}`
);
},
};
@ -658,15 +663,8 @@ async function main() {
program.option("--publish", "publish artifacts to remote registries");
program.option("--yes", "execute plan without confirmation");
program.parse(process.argv);
const {
list,
manual,
holdManual,
all,
localOnly,
publish,
yes,
} = program.opts();
const { list, manual, holdManual, all, localOnly, publish, yes } =
program.opts();
const depgraph = await getDepGraph();
if (list) {
for (const { name } of depgraph.artifacts) {

View File

@ -41,16 +41,8 @@ function makeLangScript(langConfig, isShared) {
deb,
} = install;
if (prepare) {
const {
preface,
cert,
aptKey,
aptRepo,
apt,
npm,
opam,
manual,
} = prepare;
const { preface, cert, aptKey, aptRepo, apt, npm, opam, manual } =
prepare;
if (preface) {
prefaceParts.push(preface);
}
@ -450,9 +442,11 @@ export async function generateBuildScript({ lang, type }) {
const buildScriptPath = `build/${type}/${lang}/build.bash`;
const installScriptPath = `build/${type}/${lang}/install.bash`;
await Promise.all([
fs.writeFile(buildScriptPath, buildScript + "\n")
fs
.writeFile(buildScriptPath, buildScript + "\n")
.then(() => fs.chmod(buildScriptPath, 0o755)),
fs.writeFile(installScriptPath, installScript + "\n")
fs
.writeFile(installScriptPath, installScript + "\n")
.then(() => fs.chmod(installScriptPath, 0o755)),
]);
}
@ -462,10 +456,7 @@ async function main() {
const program = new Command();
program
.requiredOption("--lang <id>", "language ID")
.requiredOption(
"--type <value>",
"package category (lang or shared)"
);
.requiredOption("--type <value>", "package category (lang or shared)");
program.parse(process.argv);
await generateBuildScript(program.opts());
process.exit(0);

View File

@ -19,11 +19,12 @@ async function getDeployConfig() {
])
)
);
const appImageTag = `app-` + await getLocalImageLabel(`riju:app`, "riju.image-hash");
const appImageTag =
`app-` + (await getLocalImageLabel(`riju:app`, "riju.image-hash"));
return {
appImageTag,
langImageTags,
}
};
}
// Parse command-line arguments, run main functionality, and exit.
@ -31,7 +32,10 @@ async function main() {
const program = new Command();
program.parse(process.argv);
await fs.mkdir("build", { recursive: true });
await fs.writeFile("build/config.json", JSON.stringify(await getDeployConfig(), null, 2) + "\n");
await fs.writeFile(
"build/config.json",
JSON.stringify(await getDeployConfig(), null, 2) + "\n"
);
console.log("wrote build/config.json");
process.exit(0);
}

View File

@ -139,9 +139,9 @@ async function encodeDockerfile(name, dependentHashes, opts) {
}
step.context = await Promise.all(
sources.map(async (source) =>
(await listFiles(source)).filter(
(entry) => !ignore.ignores(entry.path)
)
(
await listFiles(source)
).filter((entry) => !ignore.ignores(entry.path))
)
);
break;