From e2fa09df4758485c81558ccdbb3c090b2bee7fff Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 14 Mar 2021 16:12:43 -0700 Subject: [PATCH 001/104] Bugfix for provision.bash --- packer/provision.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packer/provision.bash b/packer/provision.bash index 61fe457..303041f 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -2,8 +2,8 @@ set -euo pipefail -mkdir /tmp/riju -pushd /tmp/riju +mkdir /tmp/riju-work +pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive @@ -69,4 +69,4 @@ sudo systemctl enable riju sudo passwd -l ubuntu popd -rm -rf /tmp/riju +rm -rf /tmp/riju-work From 0f3750b202dd58fe439dc96c367256339a1fb733 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 14 Mar 2021 17:18:02 -0700 Subject: [PATCH 002/104] Add some more tags --- tf/infra.tf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tf/infra.tf b/tf/infra.tf index b0c3b5b..f335b7b 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -163,14 +163,18 @@ resource "aws_instance" "server" { ami = data.aws_ami.server.id availability_zone = "${data.aws_region.current.name}b" security_groups = [aws_security_group.server.name] - tags = local.tags + tags = merge(local.tags, { + Name = "Riju server" + }) } resource "aws_ebs_volume" "data" { availability_zone = "${data.aws_region.current.name}b" size = 125 type = "gp3" - tags = local.tags + tags = merge(local.tags, { + Name = "Riju Docker data" + }) } resource "aws_volume_attachment" "data" { From 40c8c08beddcf78903d615a7b6acd10c56b130d7 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 14 Mar 2021 17:18:06 -0700 Subject: [PATCH 003/104] Make riju-init-volume more robust --- packer/riju-init-volume | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/riju-init-volume b/packer/riju-init-volume index 9219b59..520b748 100755 --- a/packer/riju-init-volume +++ b/packer/riju-init-volume @@ -10,7 +10,7 @@ mount_point=/mnt/riju/data mkdir -p "${mount_point}" -disks="$(lsblk -l -d -b -o name,size | grep nvme)" +disks="$(lsblk -l -d -b -o name,size | grep -Ev 'loop|NAME')" num_disks="$(wc -l <<< "${disks}")" if [[ "${num_disks}" != 2 ]]; then From feb5aad8f0c6149f0e536cc724e102eab3b088ee Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 18 Mar 2021 17:04:21 -0700 Subject: [PATCH 004/104] Build per-language Docker images --- Makefile | 24 +++-- docker/app/Dockerfile | 33 ++++++- .../install.bash => app/install-build.bash} | 4 + docker/app/install.bash | 1 + docker/compile/Dockerfile | 21 ---- docker/composite/Dockerfile | 18 ---- docker/composite/install.bash | 29 ------ docker/lang/Dockerfile | 14 +++ docker/lang/install.bash | 40 ++++++++ docker/runtime/install.bash | 16 +-- tools/build-composite-image.js | 97 ------------------- tools/build-lang-image.js | 73 ++++++++++++++ tools/hash-dockerfile.js | 4 +- tools/util.js | 9 ++ 14 files changed, 197 insertions(+), 186 deletions(-) rename docker/{compile/install.bash => app/install-build.bash} (95%) delete mode 100644 docker/compile/Dockerfile delete mode 100644 docker/composite/Dockerfile delete mode 100755 docker/composite/install.bash create mode 100644 docker/lang/Dockerfile create mode 100755 docker/lang/install.bash delete mode 100644 tools/build-composite-image.js create mode 100644 tools/build-lang-image.js diff --git a/Makefile b/Makefile index 07b1ec9..405ab6e 100644 --- a/Makefile +++ b/Makefile @@ -36,10 +36,11 @@ endif ## Pass NC=1 to disable the Docker cache. Base images are not pulled; ## see 'make pull-base' for that. -image: # I= [NC=1] : Build a Docker image +image: # I= [L=] [NC=1] : Build a Docker image @: $${I} -ifeq ($(I),composite) - node tools/build-composite-image.js +ifeq ($(I),lang) + @: $${L} + node tools/build-lang-image.js --lang $(L) else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else @@ -59,14 +60,23 @@ endif SHELL_ENV := -e Z -e CI -e TEST_PATIENCE -e TEST_CONCURRENCY -shell: # I= [E=1] [P1|P2=] : Launch Docker image with shell +ifeq ($(I),lang) +LANG_TAG := lang-$(L) +else +LANG_TAG := $(I) +endif + +shell: # I= [L=] [E=1] [P1|P2=] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD) -else ifneq (,$(filter $(I),compile app)) +else ifeq ($(I),app) docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) -else ifneq (,$(filter $(I),runtime composite)) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) +else ifneq (,$(filter $(I),runtime lang)) +ifeq ($(I),lang) + @: $${L} +endif + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD) else docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) endif diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index ae4ae8f..04f0cfa 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,12 +1,37 @@ -FROM riju:compile AS compile -FROM riju:composite +FROM ubuntu:rolling AS build +COPY docker/app/install-build.bash /tmp/ +RUN /tmp/install-build.bash + +WORKDIR /src +COPY Makefile ./ + +COPY system ./system/ +RUN make system + +COPY package.json yarn.lock ./ +RUN yarn install + +COPY webpack.config.cjs ./ +COPY frontend/src ./frontend/src/ +RUN make frontend + +COPY frontend/pages ./frontend/pages/ +COPY frontend/styles ./frontend/styles/ +COPY backend ./backend/ + +FROM ubuntu:rolling + +COPY docker/app/install.bash /tmp/ +RUN /tmp/install.bash + +COPY docker/shared/my_init /usr/local/sbin/ ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] + RUN useradd -p '!' -m -l -s /usr/bin/bash riju WORKDIR /src - -COPY --chown=riju:riju --from=compile /src ./ +COPY --chown=riju:riju --from=build /src ./ RUN chown root:riju system/out/*-privileged && chmod a=,g=rx,u=rwxs system/out/*-privileged USER riju diff --git a/docker/compile/install.bash b/docker/app/install-build.bash similarity index 95% rename from docker/compile/install.bash rename to docker/app/install-build.bash index 3bc48a4..ab3363f 100755 --- a/docker/compile/install.bash +++ b/docker/app/install-build.bash @@ -24,3 +24,7 @@ EOF apt-get update apt-get install -y clang g++ make nodejs sudo yarn + +rm -rf /var/lib/apt/lists/* + +rm "$0" diff --git a/docker/app/install.bash b/docker/app/install.bash index abaea1f..d23776a 100755 --- a/docker/app/install.bash +++ b/docker/app/install.bash @@ -6,6 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get dist-upgrade -y + apt-get install -y curl gnupg lsb-release curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - diff --git a/docker/compile/Dockerfile b/docker/compile/Dockerfile deleted file mode 100644 index 43490cc..0000000 --- a/docker/compile/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ubuntu:rolling - -COPY docker/compile/install.bash /tmp/ -RUN /tmp/install.bash - -WORKDIR /src -COPY Makefile ./ - -COPY system ./system/ -RUN make system - -COPY package.json yarn.lock ./ -RUN yarn install - -COPY webpack.config.cjs ./ -COPY frontend/src ./frontend/src/ -RUN make frontend - -COPY frontend/pages ./frontend/pages/ -COPY frontend/styles ./frontend/styles/ -COPY backend ./backend/ diff --git a/docker/composite/Dockerfile b/docker/composite/Dockerfile deleted file mode 100644 index 787dcca..0000000 --- a/docker/composite/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM riju:runtime - -COPY docker/composite/install.bash /tmp/ - -# The number of commands here must match NUM_SHARDS in -# build-composite-image.js. -RUN /tmp/install.bash 0 -RUN /tmp/install.bash 1 -RUN /tmp/install.bash 2 -RUN /tmp/install.bash 3 -RUN /tmp/install.bash 4 -RUN /tmp/install.bash 5 -RUN /tmp/install.bash 6 -RUN /tmp/install.bash 7 -RUN /tmp/install.bash 8 -RUN /tmp/install.bash 9 - -RUN rm /tmp/install.bash diff --git a/docker/composite/install.bash b/docker/composite/install.bash deleted file mode 100755 index 15b0ea6..0000000 --- a/docker/composite/install.bash +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -shard="$1" - -function riju-curl { - curl -fsSL "localhost:8487$1" -} - -function riju-apt-install { - riju-curl "$1" > "$(basename "$1")" - apt-get install -y "./$(basename "$1")" -} - -pushd /tmp - -export DEBIAN_FRONTEND=noninteractive - -apt-get update - -riju-curl "/shard/${shard}" | while read path; do - riju-apt-install "/fs/${path}" -done - -rm -rf *.deb -rm -rf /var/lib/apt/lists/* - -popd diff --git a/docker/lang/Dockerfile b/docker/lang/Dockerfile new file mode 100644 index 0000000..e7504df --- /dev/null +++ b/docker/lang/Dockerfile @@ -0,0 +1,14 @@ +FROM riju:runtime + +ARG LANG + +COPY docker/lang/install.bash /tmp/ +RUN /tmp/install.bash + +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] +RUN rm /usr/local/sbin/pid1.bash + +RUN useradd -p '!' -m -l -s /usr/bin/bash riju +WORKDIR /home/riju/src +USER riju +CMD ["bash"] diff --git a/docker/lang/install.bash b/docker/lang/install.bash new file mode 100755 index 0000000..c0301e7 --- /dev/null +++ b/docker/lang/install.bash @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${LANG}" + +mkdir /tmp/riju-work +pushd /tmp/riju-work + +function riju-curl { + echo >&2 "fetching ./$1" + curl -fsSL "localhost:8487/fs/$1" +} + +export DEBIAN_FRONTEND=noninteractive + +riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" + +( + dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | + (grep -Eo 'riju-shared-[^, ]+' || true) | + sed 's/riju-shared-//' +) | while read name; do + riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb" +done + +if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then + apt-get update +fi + +for file in ./riju-shared-*.deb; do + apt-get install -y "${file}" +done + +apt-get install -y "./riju-lang-${LANG}.deb" + +popd +rm -rf /tmp/riju-work + +rm "$0" diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 06f2fad..b988373 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -90,14 +90,14 @@ EOF # Unfortunately, the Microsoft repo includes a duplicate version of # the libodbc1 package whose version is not in sync with the one # shipped by the corresponding release of Ubuntu. If this one happens -# to be newer, then it'll cause a horrifyingly difficult to diagnose -# error later on while building the composite image because there's a -# conflict between the default-available versions of libodbc1 and -# libodbc1:i386, which surfaces as an inability to install -# dependencies for Erlang. Thanks Microsoft. Please don't. Anyway, -# solution is to pin this repository at a lower priority than the -# Ubuntu standard packages, so the correct version of libodbc1 gets -# installed by default. +# to be newer, then it can cause horrifyingly difficult to diagnose +# errors later on because there's a conflict between the +# default-available versions of libodbc1 and libodbc1:i386, which has +# in the past surfaced as an inability to install dependencies for +# Erlang. Thanks Microsoft. Please don't. Anyway, solution is to pin +# this repository at a lower priority than the Ubuntu standard +# packages, so the correct version of libodbc1 gets installed by +# default. tee -a /etc/apt/preferences.d/riju >/dev/null < { - res.send( - shards[parseInt(req.params.shard)] - .map(({ debPath }) => debPath + "\n") - .join("") - ); - }); - app.use("/fs", express.static(".")); - return http.createServer(app); -} - -// Given a list of the packages to be built, split them into shards. -// Return a list of shards. Each shard is a list of the package -// objects, such that there are NUM_SHARDS shards. Traversing each -// shard in order will return the packages in the same order as the -// original list. -// -// Currently this uses an extremely simple algorithm, but that might -// be improved in the future. -function getShards(pkgs) { - const shards = []; - for (let i = 0; i < NUM_SHARDS; ++i) { - shards.push([]); - } - const shardSize = Math.ceil(pkgs.length / NUM_SHARDS); - for (let i = 0; i < pkgs.length; ++i) { - shards[Math.floor(i / shardSize)].push(pkgs[i]); - } - return shards; -} - -// Parse command-line arguments, run main functionality, and exit. -async function main() { - const packages = await getPackages(); - const hash = await hashDockerfile( - "composite", - { - "riju:runtime": await getLocalImageLabel( - "riju:runtime", - "riju.image-hash" - ), - }, - { - salt: { - packageHashes: ( - await Promise.all( - packages.map(async ({ debPath }) => { - return ( - await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { - getStdout: true, - }) - ).stdout.trim(); - }) - ) - ).sort(), - }, - } - ); - const server = getServer({ - shards: getShards(packages), - }); - await new Promise((resolve) => server.listen(8487, "localhost", resolve)); - try { - await runCommand( - `docker build . -f docker/composite/Dockerfile -t riju:composite` + - ` --network host --no-cache --label riju.image-hash=${hash}` - ); - } finally { - await server.close(); - } - process.exit(0); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js new file mode 100644 index 0000000..e7c8e4b --- /dev/null +++ b/tools/build-lang-image.js @@ -0,0 +1,73 @@ +import { promises as fs } from "fs"; +import http from "http"; + +import { Command } from "commander"; +import express from "express"; + +import { readLangConfig } from "./config.js"; +import { getLocalImageLabel } from "./docker-util.js"; +import { hashDockerfile } from "./hash-dockerfile.js"; +import { getDebHash, runCommand } from "./util.js"; + +// Get a Node.js http server object that will allow the Docker +// build to fetch files from outside the container, without them +// being in the build context. +function getServer() { + const app = express(); + app.use("/fs", express.static(".")); + return http.createServer(app); +} + +// Parse command-line arguments, run main functionality, and exit. +async function main() { + const program = new Command(); + program.requiredOption("--lang ", "language ID"); + program.option("--debug", "interactive debugging"); + program.parse(process.argv); + const { lang, debug } = program; + const hash = await hashDockerfile( + "lang", + { + "riju:runtime": await getLocalImageLabel( + "riju:runtime", + "riju.image-hash" + ), + }, + { + salt: { + langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`), + sharedHashes: ( + await Promise.all( + (((await readLangConfig(lang)).install || {}).riju || []).map( + async (name) => + await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`) + ) + ) + ).sort(), + }, + } + ); + const server = getServer(); + await new Promise((resolve) => server.listen(8487, "localhost", resolve)); + try { + if (debug) { + await runCommand( + `docker run -it --rm -e LANG=${lang} -w /tmp/riju-work --network host riju:runtime` + ); + } else { + await runCommand( + `docker build . -f docker/lang/Dockerfile ` + + `--build-arg LANG=${lang} -t riju:lang-${lang} ` + + `--network host --no-cache --label riju.image-hash=${hash}` + ); + } + } finally { + await server.close(); + } + process.exit(0); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tools/hash-dockerfile.js b/tools/hash-dockerfile.js index 74992d9..bef6be8 100644 --- a/tools/hash-dockerfile.js +++ b/tools/hash-dockerfile.js @@ -187,8 +187,8 @@ async function main() { } const [name] = program.args; const { debug } = program.opts(); - if (name === "composite") { - throw new Error("use build-composite-image.js instead for this"); + if (name === "lang") { + throw new Error("use build-lang-image.js instead for this"); } if (debug) { console.log( diff --git a/tools/util.js b/tools/util.js index 900dc01..dfc29a3 100644 --- a/tools/util.js +++ b/tools/util.js @@ -43,3 +43,12 @@ export async function runCommand(cmd, options) { } return rv; } + +// Return the Riju-Script-Hash field in a .deb file generated by Riju. +export async function getDebHash(debPath) { + return ( + await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { + getStdout: true, + }) + ).stdout.trim(); +} From 83208355d40b85b82d47b1132b9f22ccdd559d55 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 18 Mar 2021 19:03:28 -0700 Subject: [PATCH 005/104] Load language configurations into server --- .dockerignore | 1 + .gitignore | 3 +++ Makefile | 6 +++--- backend/langs.js | 25 ++++++++----------------- docker/app/Dockerfile | 2 ++ tools/config.js => lib/yaml.js | 0 tools/build-lang-image.js | 12 +++++++----- tools/generate-build-script.js | 2 +- tools/make-foreach.js | 2 +- tools/plan-publish.js | 2 +- tools/write-all-build-scripts.js | 2 +- 11 files changed, 28 insertions(+), 29 deletions(-) rename tools/config.js => lib/yaml.js (100%) diff --git a/.dockerignore b/.dockerignore index 0cf0b25..782411e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,6 @@ **/.lsp-repl-history **/.terraform **/build +**/build-docker **/node_modules **/out diff --git a/.gitignore b/.gitignore index 84d2869..382d670 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,8 @@ .lsp-repl-history .terraform build +# Separate directory for things that are ignored by Git but not by +# Docker. +build-docker node_modules out diff --git a/Makefile b/Makefile index 405ab6e..04b2536 100644 --- a/Makefile +++ b/Makefile @@ -256,11 +256,11 @@ publish: # Full synchronization and prod deployment ### Miscellaneous -## Run this every time you update .gitignore. +## Run this every time you update .gitignore or .dockerignore.in. -dockerignore: # Update .dockerignore from .gitignore +dockerignore: # Update .dockerignore from .gitignore and .dockerignore.in echo "# This file is generated by 'make dockerignore', do not edit." > .dockerignore - cat .gitignore | sed 's#^#**/#' >> .dockerignore + cat .gitignore | sed 's/#.*//' | grep . | sed 's#^#**/#' >> .dockerignore ## You need to be inside a 'make env' shell whenever you are running ## manual commands (Docker, Terraform, Packer, etc.) directly, as diff --git a/backend/langs.js b/backend/langs.js index c1f35d3..63716b0 100644 --- a/backend/langs.js +++ b/backend/langs.js @@ -3,6 +3,7 @@ import path from "path"; import debounce from "debounce"; +import { getLangs, readLangConfig } from "../lib/yaml.js"; import { log } from "./util.js"; // Map from language IDs to language configuration objects. This is @@ -16,24 +17,14 @@ export let aliases = {}; // global langs variable in this module. Never throw an error. If // there is a problem then just leave the languages as they previously // were. -async function readLangsFromDisk() { +async function updateLangsFromDisk() { try { const newLangs = {}; const newAliases = {}; - for (const filename of await fs.readdir("/opt/riju/langs")) { - if (path.parse(filename).ext !== ".json") { - continue; - } - const id = path.parse(filename).name; - const langConfig = JSON.parse( - await fs.readFile(`/opt/riju/langs/${filename}`, "utf-8") - ); - if (langConfig.id !== id) { - log.error( - "Language config ${filename} has mismatched language ID ${id}, ignoring" - ); - continue; - } + for (const langConfig of await Promise.all( + (await getLangs()).map(readLangConfig) + )) { + const { id } = langConfig; newLangs[id] = langConfig; newAliases[id] = id; for (const alias of langConfig.aliases || []) { @@ -52,6 +43,6 @@ async function readLangsFromDisk() { } } -export const langsPromise = readLangsFromDisk().then(() => langs); +export const langsPromise = updateLangsFromDisk().then(() => langs); -fsOrig.watch("/opt/riju/langs", debounce(readLangsFromDisk, 200)); +fsOrig.watch("langs", debounce(updateLangsFromDisk, 200)); diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 04f0cfa..9d27f1d 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -18,7 +18,9 @@ RUN make frontend COPY frontend/pages ./frontend/pages/ COPY frontend/styles ./frontend/styles/ +COPY lib ./lib/ COPY backend ./backend/ +COPY langs ./langs/ FROM ubuntu:rolling diff --git a/tools/config.js b/lib/yaml.js similarity index 100% rename from tools/config.js rename to lib/yaml.js diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js index e7c8e4b..3e5ab36 100644 --- a/tools/build-lang-image.js +++ b/tools/build-lang-image.js @@ -4,7 +4,7 @@ import http from "http"; import { Command } from "commander"; import express from "express"; -import { readLangConfig } from "./config.js"; +import { readLangConfig } from "../lib/yaml.js"; import { getLocalImageLabel } from "./docker-util.js"; import { hashDockerfile } from "./hash-dockerfile.js"; import { getDebHash, runCommand } from "./util.js"; @@ -67,7 +67,9 @@ async function main() { process.exit(0); } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +if (process.argv[1] === url.fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index d6dc8fc..df6dfb6 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -5,7 +5,7 @@ import url from "url"; import { Command } from "commander"; import YAML from "yaml"; -import { readLangConfig, readSharedDepConfig } from "./config.js"; +import { readLangConfig, readSharedDepConfig } from "../lib/yaml.js"; // Given a language config object, return the text of a Bash script // that will build the (unpacked) riju-lang-foo Debian package into diff --git a/tools/make-foreach.js b/tools/make-foreach.js index c52a7ac..56fc1e5 100644 --- a/tools/make-foreach.js +++ b/tools/make-foreach.js @@ -1,7 +1,7 @@ import process from "process"; import url from "url"; -import { getPackages } from "./config.js"; +import { getPackages } from "../lib/yaml.js"; import { runCommand } from "./util.js"; // Parse command-line arguments, run main functionality, and exit. diff --git a/tools/plan-publish.js b/tools/plan-publish.js index d8149c6..fa389dc 100644 --- a/tools/plan-publish.js +++ b/tools/plan-publish.js @@ -7,7 +7,7 @@ import { Command } from "commander"; import _ from "lodash"; import { v4 as getUUID } from "uuid"; -import { getLangs, getPackages, readLangConfig } from "./config.js"; +import { getLangs, getPackages, readLangConfig } from "../lib/yaml.js"; import { getLocalImageDigest, getLocalImageLabel, diff --git a/tools/write-all-build-scripts.js b/tools/write-all-build-scripts.js index 14573f1..06868b5 100644 --- a/tools/write-all-build-scripts.js +++ b/tools/write-all-build-scripts.js @@ -7,7 +7,7 @@ import nodePath from "path"; import process from "process"; import url from "url"; -import { getPackages } from "./config.js"; +import { getPackages } from "../lib/yaml.js"; import { generateBuildScript } from "./generate-build-script.js"; // Parse command-line arguments, run main functionality, and exit. From b99d17bcd30ec2b67f507649f31972c3271201ed Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 19 Mar 2021 23:04:59 -0700 Subject: [PATCH 006/104] Preliminary containerization work --- Makefile | 4 +- backend/api.js | 90 +++++++----- backend/sandbox.js | 11 +- backend/test-runner.js | 2 +- backend/users.js | 115 --------------- backend/util.js | 78 ++-------- docker/app/Dockerfile | 9 +- docker/app/install.bash | 30 ---- docker/base/Dockerfile | 9 ++ docker/base/install.bash | 164 +++++++++++++++++++++ docker/lang/Dockerfile | 2 +- docker/packaging/install.bash | 4 +- docker/runtime/Dockerfile | 3 +- docker/runtime/install.bash | 113 ++------------- system/compile.bash | 6 +- system/src/riju-system-privileged.c | 217 +++++++++++++--------------- tools/build-lang-image.js | 7 +- tools/hash-dockerfile.js | 2 +- tools/plan-publish.js | 12 +- 19 files changed, 368 insertions(+), 510 deletions(-) delete mode 100644 backend/users.js delete mode 100755 docker/app/install.bash create mode 100644 docker/base/Dockerfile create mode 100755 docker/base/install.bash diff --git a/Makefile b/Makefile index 04b2536..caf0d99 100644 --- a/Makefile +++ b/Makefile @@ -72,11 +72,13 @@ ifneq (,$(filter $(I),admin ci)) docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) -else ifneq (,$(filter $(I),runtime lang)) +else ifneq (,$(filter $(I),base lang)) ifeq ($(I),lang) @: $${L} endif docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD) +else ifeq ($(I),runtime) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) else docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) endif diff --git a/backend/api.js b/backend/api.js index ddc16ad..86cdb0c 100644 --- a/backend/api.js +++ b/backend/api.js @@ -6,12 +6,10 @@ import pty from "node-pty"; import pQueue from "p-queue"; const PQueue = pQueue.default; import rpc from "vscode-jsonrpc"; -import { v4 as getUUID } from "uuid"; import { langs } from "./langs.js"; -import { borrowUser } from "./users.js"; import * as util from "./util.js"; -import { bash } from "./util.js"; +import { bash, getUUID } from "./util.js"; const allSessions = new Set(); @@ -24,16 +22,8 @@ export class Session { return langs[this.lang]; } - get uid() { - return this.uidInfo.uid; - } - - returnUser = async () => { - this.uidInfo && (await this.uidInfo.returnUser()); - }; - get context() { - return { uid: this.uid, uuid: this.uuid }; + return { uuid: this.uuid, lang: this.lang }; } log = (msg) => this.logPrimitive(`[${this.uuid}] ${msg}`); @@ -43,7 +33,7 @@ export class Session { this.uuid = getUUID(); this.lang = lang; this.tearingDown = false; - this.uidInfo = null; + this.container = null; this.term = null; this.lsp = null; this.daemon = null; @@ -57,24 +47,48 @@ export class Session { return await util.run(args, this.log, options); }; - privilegedSetup = () => util.privilegedSetup(this.context); - privilegedSpawn = (args) => util.privilegedSpawn(this.context, args); - privilegedUseradd = () => util.privilegedUseradd(this.uid); - privilegedTeardown = () => util.privilegedTeardown(this.context); + privilegedSession = () => util.privilegedSession(this.context); + privilegedWait = () => util.privilegedWait(this.context); + privilegedExec = (args) => util.privilegedExec(this.context, args); setup = async () => { try { allSessions.add(this); - const { uid, returnUser } = await borrowUser(); - this.uidInfo = { uid, returnUser }; - this.log(`Borrowed uid ${this.uid}`); - await this.run(this.privilegedSetup()); + const containerArgs = this.privilegedSession(); + const containerProc = spawn(containerArgs[0], containerArgs.slice(1)); + this.container = { + proc: containerProc, + }; + for (const stream of [containerProc.stdout, containerProc.stderr]) { + stream.on("data", (data) => + this.send({ + event: "serviceLog", + service: "container", + output: data.toString("utf8"), + }) + ); + containerProc.on("close", (code, signal) => + this.send({ + event: "serviceFailed", + service: "container", + error: `Exited with status ${signal || code}`, + }) + ); + containerProc.on("error", (err) => + this.send({ + event: "serviceFailed", + service: "container", + error: `${err}`, + }) + ); + } + await this.run(this.privilegedWait(this.context)); if (this.config.setup) { - await this.run(this.privilegedSpawn(bash(this.config.setup))); + await this.run(this.privilegedExec(bash(this.config.setup))); } await this.runCode(); if (this.config.daemon) { - const daemonArgs = this.privilegedSpawn(bash(this.config.daemon)); + const daemonArgs = this.privilegedExec(bash(this.config.daemon)); const daemonProc = spawn(daemonArgs[0], daemonArgs.slice(1)); this.daemon = { proc: daemonProc, @@ -105,9 +119,9 @@ export class Session { } if (this.config.lsp) { if (this.config.lsp.setup) { - await this.run(this.privilegedSpawn(bash(this.config.lsp.setup))); + await this.run(this.privilegedExec(bash(this.config.lsp.setup))); } - const lspArgs = this.privilegedSpawn(bash(this.config.lsp.start)); + const lspArgs = this.privilegedExec(bash(this.config.lsp.start)); const lspProc = spawn(lspArgs[0], lspArgs.slice(1)); this.lsp = { proc: lspProc, @@ -252,7 +266,7 @@ export class Session { writeCode = async (code) => { if (this.config.main.includes("/")) { await this.run( - this.privilegedSpawn([ + this.privilegedExec([ "mkdir", "-p", path.dirname(`${this.homedir}/${this.config.main}`), @@ -260,7 +274,7 @@ export class Session { ); } await this.run( - this.privilegedSpawn([ + this.privilegedExec([ "sh", "-c", `cat > ${path.resolve(this.homedir, this.config.main)}`, @@ -283,7 +297,7 @@ export class Session { } = this.config; if (this.term) { const pid = this.term.pty.pid; - const args = this.privilegedSpawn( + const args = this.privilegedExec( bash(`kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}`) ); spawn(args[0], args.slice(1)); @@ -310,7 +324,7 @@ export class Session { code += suffix + "\n"; } await this.writeCode(code); - const termArgs = this.privilegedSpawn(bash(cmdline)); + const termArgs = this.privilegedExec(bash(cmdline)); const term = { pty: pty.spawn(termArgs[0], termArgs.slice(1), { name: "xterm-color", @@ -349,14 +363,14 @@ export class Session { } if (this.formatter) { const pid = this.formatter.proc.pid; - const args = this.privilegedSpawn( + const args = this.privilegedExec( bash(`kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}`) ); spawn(args[0], args.slice(1)); this.formatter.live = false; this.formatter = null; } - const args = this.privilegedSpawn(bash(this.config.format.run)); + const args = this.privilegedExec(bash(this.config.format.run)); const formatter = { proc: spawn(args[0], args.slice(1)), live: true, @@ -409,7 +423,7 @@ export class Session { }; ensure = async (cmd) => { - const code = await this.run(this.privilegedSpawn(bash(cmd)), { + const code = await this.run(this.privilegedExec(bash(cmd)), { check: false, }); this.send({ event: "ensured", code }); @@ -422,11 +436,15 @@ export class Session { } this.log(`Tearing down session`); this.tearingDown = true; - allSessions.delete(this); - if (this.uidInfo) { - await this.run(this.privilegedTeardown()); - await this.returnUser(); + if (this.container) { + // SIGTERM should be sufficient as the command running in the + // foreground is just 'tail -f /dev/null' which won't try to + // block signals. Killing the foreground process (i.e. pid1) + // should cause the Docker runtime to bring everything else + // down in flames. + this.container.proc.kill(); } + allSessions.delete(this); this.ws.terminate(); } catch (err) { this.log(`Error during teardown`); diff --git a/backend/sandbox.js b/backend/sandbox.js index 4dddd9b..a89d79e 100644 --- a/backend/sandbox.js +++ b/backend/sandbox.js @@ -3,9 +3,9 @@ import { promises as fs } from "fs"; import process from "process"; import { quote } from "shell-quote"; -import { v4 as getUUID } from "uuid"; -import { borrowUser } from "./users.js"; +import { getUUID } from "./util.js"; + import { privilegedSetup, privilegedSpawn, @@ -29,9 +29,8 @@ async function main() { die("environment variable unset: $L"); } const uuid = getUUID(); - const { uid, returnUser } = await borrowUser(log); - await run(privilegedSetup({ uid, uuid }), log); - const args = privilegedSpawn({ uid, uuid }, [ + await run(privilegedSetup({ uuid }), log); + const args = privilegedSpawn({ uuid }, [ "bash", "-c", `exec env L='${lang}' bash --rcfile <(cat <<< ${quote([sandboxScript])})`, @@ -43,7 +42,7 @@ async function main() { proc.on("error", reject); proc.on("close", resolve); }); - await run(privilegedTeardown({ uid, uuid }), log); + await run(privilegedTeardown({ uuid }), log); await returnUser(); } diff --git a/backend/test-runner.js b/backend/test-runner.js index a6de2ce..86a55bc 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -5,10 +5,10 @@ import _ from "lodash"; import pQueue from "p-queue"; const PQueue = pQueue.default; import stripAnsi from "strip-ansi"; -import { v4 as getUUID } from "uuid"; import * as api from "./api.js"; import { langsPromise } from "./langs.js"; +import { getUUID } from "./util.js"; let langs = {}; diff --git a/backend/users.js b/backend/users.js deleted file mode 100644 index 2f9af6f..0000000 --- a/backend/users.js +++ /dev/null @@ -1,115 +0,0 @@ -import { spawn } from "child_process"; -import { promises as fs } from "fs"; -import os from "os"; - -import AsyncLock from "async-lock"; -import _ from "lodash"; -import parsePasswd from "parse-passwd"; - -import { asBool, privilegedUseradd, run, uuidRegexp } from "./util.js"; - -// Keep in sync with system/src/riju-system-privileged.c -export const MIN_UID = 2000; -export const MAX_UID = 65000; - -function validUID(uid) { - return uid >= MIN_UID && uid < MAX_UID; -} - -const CUR_UID = os.userInfo().uid; -const ASSUME_SINGLE_PROCESS = asBool( - process.env.RIJU_ASSUME_SINGLE_PROCESS, - false -); - -let initialized = false; -let nextUserToCreate = null; -let locallyBorrowedUsers = new Set(); -let availableUsers = new Set(); -let lock = new AsyncLock(); - -async function getCreatedUsers() { - return new Set( - parsePasswd(await fs.readFile("/etc/passwd", "utf-8")) - .map(({ uid }) => parseInt(uid)) - .filter((uid) => !isNaN(uid) && validUID(uid)) - ); -} - -async function getActiveUsers() { - let dirents; - try { - dirents = await fs.readdir("/tmp/riju"); - } catch (err) { - if (err.code === "ENOENT") { - return new Set(); - } - throw err; - } - return new Set( - ( - await Promise.all( - dirents - .filter((name) => name.match(uuidRegexp)) - .map((name) => fs.stat(`/tmp/riju/${name}`)) - ) - ) - .map(({ uid }) => uid) - .filter(validUID) - ); -} - -async function createUser(log) { - if (nextUserToCreate >= MAX_UID) { - throw new Error("too many users"); - } - const uid = nextUserToCreate; - await run(privilegedUseradd(uid), log); - nextUserToCreate += 1; - return uid; -} - -export async function borrowUser(log) { - return await lock.acquire("key", async () => { - if (!initialized || !ASSUME_SINGLE_PROCESS) { - const createdUsers = await getCreatedUsers(); - const activeUsers = await getActiveUsers(); - if (createdUsers.size > 0) { - nextUserToCreate = _.max([...createdUsers]) + 1; - } else { - nextUserToCreate = MIN_UID; - } - // If there are new users created, we want to make them - // available (unless they are already active). Similarly, if - // there are users that have become inactive, we want to make - // them available (unless they are already borrowed locally). - for (const user of createdUsers) { - if (!activeUsers.has(user) && !locallyBorrowedUsers.has(user)) { - availableUsers.add(user); - } - } - // If there are users that have become active, we want to make - // them unavailable. - for (const user of activeUsers) { - availableUsers.delete(user); - } - initialized = true; - } - if (availableUsers.size === 0) { - availableUsers.add(await createUser(log)); - } - // https://stackoverflow.com/a/32539929/3538165 - const user = availableUsers.values().next().value; - locallyBorrowedUsers.add(user); - availableUsers.delete(user); - return { - uid: user, - returnUser: async () => { - await lock.acquire("key", () => { - locallyBorrowedUsers.delete(user); - availableUsers.add(user); - }); - }, - }; - }); -} diff --git a/backend/util.js b/backend/util.js index 69643c3..ce29b8a 100644 --- a/backend/util.js +++ b/backend/util.js @@ -3,55 +3,12 @@ import os from "os"; import process from "process"; import { quote } from "shell-quote"; - -import { MIN_UID, MAX_UID } from "./users.js"; +import { v4 as getUUIDOrig } from "uuid"; export const rijuSystemPrivileged = "system/out/riju-system-privileged"; -const rubyVersion = (() => { - try { - return spawnSync("ruby", ["-e", "puts RUBY_VERSION"]) - .stdout.toString() - .trim(); - } catch (err) { - return null; - } -})(); - -function getEnv({ uid, uuid }) { - const cwd = `/tmp/riju/${uuid}`; - const path = [ - rubyVersion && `${cwd}/.gem/ruby/${rubyVersion}/bin`, - `${cwd}/.local/bin`, - `${cwd}/node_modules/.bin`, - `/usr/local/sbin`, - `/usr/local/bin`, - `/usr/sbin`, - `/usr/bin`, - `/bin`, - ].filter((x) => x); - const username = - uid >= MIN_UID && uid < MAX_UID ? `riju${uid}` : os.userInfo().username; - return { - HOME: cwd, - HOSTNAME: "riju", - LANG: "C.UTF-8", - LC_ALL: "C.UTF-8", - LOGNAME: username, - PATH: path.join(":"), - PWD: cwd, - SHELL: "/usr/bin/bash", - TERM: "xterm-256color", - TMPDIR: `${cwd}`, - USER: username, - USERNAME: username, - }; -} - -function getEnvString(ctx) { - return Object.entries(getEnv(ctx)) - .map(([key, val]) => `${key}=${quote([val])}`) - .join(" "); +export function getUUID() { + return getUUIDOrig().replace(/-/g, ""); } export async function run(args, log, options) { @@ -87,30 +44,16 @@ export async function run(args, log, options) { }); } -export function privilegedUseradd(uid) { - return [rijuSystemPrivileged, "useradd", `${uid}`]; +export function privilegedSession({ uuid, lang }) { + return [rijuSystemPrivileged, "session", uuid, lang]; } -export function privilegedSetup({ uid, uuid }) { - return [rijuSystemPrivileged, "setup", `${uid}`, uuid]; +export function privilegedWait({ uuid }) { + return [rijuSystemPrivileged, "wait", uuid]; } -export function privilegedSpawn(ctx, args) { - const { uid, uuid } = ctx; - return [ - rijuSystemPrivileged, - "spawn", - `${uid}`, - uuid, - "sh", - "-c", - `exec env -i ${getEnvString(ctx)} "$@"`, - "--", - ].concat(args); -} - -export function privilegedTeardown({ uid, uuid }) { - return [rijuSystemPrivileged, "teardown", `${uid}`, uuid]; +export function privilegedExec({ uuid }, args) { + return [rijuSystemPrivileged, "exec", uuid].concat(args); } export function bash(cmdline) { @@ -130,9 +73,6 @@ export const log = { error: console.error, }; -// https://gist.github.com/bugventure/f71337e3927c34132b9a -export const uuidRegexp = /^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/; - export function asBool(value, def) { if (def === undefined) { throw new Error("asBool needs an explicit default value"); diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 9d27f1d..cd36ed1 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -22,17 +22,10 @@ COPY lib ./lib/ COPY backend ./backend/ COPY langs ./langs/ -FROM ubuntu:rolling +FROM riju:runtime -COPY docker/app/install.bash /tmp/ -RUN /tmp/install.bash - -COPY docker/shared/my_init /usr/local/sbin/ ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] - RUN useradd -p '!' -m -l -s /usr/bin/bash riju - -WORKDIR /src COPY --chown=riju:riju --from=build /src ./ RUN chown root:riju system/out/*-privileged && chmod a=,g=rx,u=rwxs system/out/*-privileged diff --git a/docker/app/install.bash b/docker/app/install.bash deleted file mode 100755 index d23776a..0000000 --- a/docker/app/install.bash +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get dist-upgrade -y - -apt-get install -y curl gnupg lsb-release - -curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - -curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - - -ubuntu_ver="$(lsb_release -rs)" -ubuntu_name="$(lsb_release -cs)" - -node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" - -tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null </dev/null </dev/null <<"EOF" +%sudo ALL=(ALL:ALL) NOPASSWD: ALL +EOF + +popd +rm -rf /tmp/riju-work + +rm "$0" diff --git a/docker/lang/Dockerfile b/docker/lang/Dockerfile index e7504df..f38b4f8 100644 --- a/docker/lang/Dockerfile +++ b/docker/lang/Dockerfile @@ -1,4 +1,4 @@ -FROM riju:runtime +FROM riju:base ARG LANG diff --git a/docker/packaging/install.bash b/docker/packaging/install.bash index 6d47b02..1b08ef3 100755 --- a/docker/packaging/install.bash +++ b/docker/packaging/install.bash @@ -2,8 +2,8 @@ set -euxo pipefail -# See install.bash for the runtime image for much of the same, but -# with more comments. +# See install.bash for the base image for much of the same, but with +# more comments. mkdir /tmp/riju-work pushd /tmp/riju-work diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile index 76e0872..62acade 100644 --- a/docker/runtime/Dockerfile +++ b/docker/runtime/Dockerfile @@ -3,9 +3,10 @@ FROM ubuntu:rolling COPY docker/runtime/install.bash /tmp/ RUN /tmp/install.bash -WORKDIR /src COPY docker/shared/my_init docker/runtime/pid1.bash /usr/local/sbin/ ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--", "/usr/local/sbin/pid1.bash"] + +WORKDIR /src CMD ["bash"] EXPOSE 6119 EXPOSE 6120 diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index b988373..87da6ad 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -11,120 +11,32 @@ pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive -dpkg --add-architecture i386 - apt-get update (yes || true) | unminimize apt-get install -y curl gnupg lsb-release wget -# Ceylon -wget https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem -O /usr/local/share/ca-certificates/DigiCertTLSRSASHA2562020CA1.crt +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - +curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -# D -wget https://letsencrypt.org/certs/lets-encrypt-r3.pem -O /usr/local/share/ca-certificates/lets-encrypt-r3.crt - -update-ca-certificates - -ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" -cran_repo="$(curl -fsSL https://cran.r-project.org/bin/linux/ubuntu/ | grep -Eo 'cran[0-9]+' | head -n1)" -node_repo="$(curl -fsSL https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" - -# .NET -wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" -apt-get install ./packages-microsoft-prod.deb - -# Ceylon -curl -fsSL https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key | apt-key add - - -# Crystal -curl -fsSL https://keybase.io/crystal/pgp_keys.asc | apt-key add - - -# Dart -curl -fsSL https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - - -# Hack -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B4112585D386EB94 - -# MongoDB -curl -fsSL https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - - -# Node.js -curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - - -# R -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 - -# Yarn -curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null </dev/null <<"EOF" %sudo ALL=(ALL:ALL) NOPASSWD: ALL EOF -mkdir -p /opt/riju/langs -touch /opt/riju/langs/.keep - popd rm -rf /tmp/riju-work diff --git a/system/compile.bash b/system/compile.bash index e7c1cae..3907274 100755 --- a/system/compile.bash +++ b/system/compile.bash @@ -19,9 +19,7 @@ for src in system/src/*.c; do out="${out/.c}" verbosely clang -Wall -Wextra -Werror -std=c11 "${src}" -o "${out}" if [[ "${out}" == *-privileged ]]; then - if getent group riju >/dev/null; then - sudo chown root:riju "${out}" - fi - sudo chmod a=,g=rx,u=rwxs "${out}" + verbosely sudo chown root:riju "${out}" + verbosely sudo chmod a=,g=rx,u=rwxs "${out}" fi done diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index b6dfd6a..44a0f6b 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -1,19 +1,15 @@ #define _GNU_SOURCE #include #include +#include #include #include #include #include #include +#include #include -// Keep in sync with backend/src/users.ts -const int MIN_UID = 2000; -const int MAX_UID = 65000; - -int privileged; - void __attribute__ ((noreturn)) die(char *msg) { fprintf(stderr, "%s\n", msg); @@ -23,155 +19,136 @@ void __attribute__ ((noreturn)) die(char *msg) void die_with_usage() { die("usage:\n" - " riju-system-privileged useradd UID\n" - " riju-system-privileged setup UID UUID\n" - " riju-system-privileged spawn UID UUID CMDLINE...\n" - " riju-system-privileged teardown UID UUID"); -} - -int parseUID(char *str) -{ - if (!privileged) - return -1; - char *endptr; - long uid = strtol(str, &endptr, 10); - if (!*str || *endptr) - die("uid must be an integer"); - if (uid < MIN_UID || uid >= MAX_UID) - die("uid is out of range"); - return uid; + " riju-system-privileged session UUID LANG\n" + " riju-system-privileged wait UUID\n" + " riju-system-privileged exec UUID CMDLINE..."); } char *parseUUID(char *uuid) { - if (!*uuid) + if (strnlen(uuid, 33) != 32) die("illegal uuid"); for (char *ptr = uuid; *ptr; ++ptr) - if (!((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= '0' && *ptr <= '9') || *ptr == '-')) + if (!((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= '0' && *ptr <= '9'))) die("illegal uuid"); return uuid; } -void useradd(int uid) -{ - if (!privileged) - die("useradd not allowed without root privileges"); - char *cmdline; - if (asprintf(&cmdline, "groupadd -g %1$d riju%1$d", uid) < 0) - die("asprintf failed"); - int status = system(cmdline); - if (status != 0) - die("groupadd failed"); - if (asprintf(&cmdline, "useradd -M -N -l -r -u %1$d -g %1$d -p '!' -s /usr/bin/bash riju%1$d", uid) < 0) - die("asprintf failed"); - status = system(cmdline); - if (status != 0) - die("useradd failed"); +char *parseLang(char *lang) { + size_t len = strnlen(lang, 65); + if (len == 0 || len > 64) + die("illegal language name"); + return lang; } -void spawn(int uid, char *uuid, char **cmdline) +void session(char *uuid, char *lang) { - char *cwd; - if (asprintf(&cwd, "/tmp/riju/%s", uuid) < 0) + char *image, *container; + if (asprintf(&image, "riju:lang-%s", lang) < 0) die("asprintf failed"); - if (chdir(cwd) < 0) - die("chdir failed"); - if (privileged) { - if (setgid(uid) < 0) - die("setgid failed"); - if (setgroups(0, NULL) < 0) - die("setgroups failed"); - if (setuid(uid) < 0) - die("setuid failed"); - } - umask(077); - execvp(cmdline[0], cmdline); + if (asprintf(&container, "riju-session-%s", uuid) < 0) + die("asprintf failed"); + char *argv[] = { + "docker", + "run", + "--rm", + "-e", "HOME=/home/riju", + "-e", "HOSTNAME=riju", + "-e", "LANG=C.UTF-8", + "-e", "LC_ALL=C.UTF-8", + "-e", "LOGNAME=riju", + "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin", + "-e", "PWD=/home/riju/src", + "-e", "SHELL=/usr/bin/bash", + "-e", "TERM=xterm-256color", + "-e", "TMPDIR=/tmp", + "-e", "USER=riju", + "-e", "USERNAME=riju", + "--hostname", "riju", + "--name", container, + image, "tail", "-f", "/dev/null", NULL, + }; + execvp(argv[0], argv); die("execvp failed"); } -void setup(int uid, char *uuid) +void wait_alarm(int signum) { - char *cmdline; - if (asprintf(&cmdline, privileged - ? "install -d -o riju%1$d -g riju%1$d -m 700 /tmp/riju/%2$s" - : "install -d -m 700 /tmp/riju/%2$s", uid, uuid) < 0) - die("asprintf failed"); - int status = system(cmdline); - if (status != 0) - die("install failed"); + (void)signum; + die("container did not come up within 1 second"); } -void teardown(int uid, char *uuid) +void wait(char *uuid) { char *cmdline; - int status; - char *users; - if (uid >= MIN_UID && uid < MAX_UID) { - if (asprintf(&users, "%d", uid) < 0) - die("asprintf failed"); - } else { - cmdline = "getent passwd | grep -Eo '^riju[0-9]{4}' | paste -s -d, - | tr -d '\n'"; - FILE *fp = popen(cmdline, "r"); - if (fp == NULL) - die("popen failed"); - static char buf[(MAX_UID - MIN_UID) * 9]; - if (fgets(buf, sizeof(buf), fp) == NULL) { - if (feof(fp)) - users = NULL; - else { - die("fgets failed"); - } - } else - users = buf; - } - if (users != NULL) { - if (asprintf(&cmdline, "while pkill -9 --uid %1$s; do sleep 0.01; done", users) < 0) - die("asprintf failed"); - status = system(cmdline); - if (status != 0 && status != 256) - die("pkill failed"); - } - if (asprintf(&cmdline, "rm -rf /tmp/riju/%s", uuid) < 0) + if (asprintf(&cmdline, "docker inspect riju-session-%s", uuid) < 0) die("asprintf failed"); - status = system(cmdline); - if (status != 0) - die("rm failed"); + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 1000 * 1000 * 10; + signal(SIGALRM, wait_alarm); + alarm(1); + while (1) { + FILE *proc = popen(cmdline, "r"); + if (proc == NULL) + die("popen failed"); + int status = pclose(proc); + if (status < 0) + die("pclose failed"); + if (WEXITSTATUS(status) == 0) + break; + int rv = nanosleep(&ts, NULL); + if (rv != 0 && rv != EINTR) + die("nanosleep failed"); + } +} + +void exec(char *uuid, int argc, char **cmdline) +{ + char *container; + if (asprintf(&container, "riju-session-%s", uuid) < 0) + die("asprintf failed"); + char *argvPrefix[] = { + "docker", + "exec", + "-it", + container, + }; + char **argv = malloc(sizeof(argvPrefix) + (argc + 1) * sizeof(char *)); + if (argv == NULL) + die("malloc failed"); + memcpy(argv, argvPrefix, sizeof(argvPrefix)); + memcpy((void *)argv + sizeof(argvPrefix), cmdline, argc * sizeof(char *)); + argv[sizeof(argvPrefix) + argc * sizeof(char *)] = NULL; + execvp(argv[0], argv); + die("execvp failed"); } int main(int argc, char **argv) { - int code = setuid(0); - if (code != 0 && code != -EPERM) + if (setuid(0) != 0) die("setuid failed"); - privileged = code == 0; if (argc < 2) die_with_usage(); - if (!strcmp(argv[1], "useradd")) { + if (!strcmp(argv[1], "session")) { + if (argc != 4) + die_with_usage(); + char *uuid = parseUUID(argv[2]); + char *lang = parseLang(argv[3]); + session(uuid, lang); + return 0; + } + if (!strcmp(argv[1], "wait")) { if (argc != 3) die_with_usage(); - useradd(parseUID(argv[2])); + char *uuid = parseUUID(argv[2]); + wait(uuid); return 0; } - if (!strcmp(argv[1], "spawn")) { - if (argc < 5) + if (!strcmp(argv[1], "exec")) { + if (argc < 4) die_with_usage(); - spawn(parseUID(argv[2]), parseUUID(argv[3]), &argv[4]); - return 0; - } - if (!strcmp(argv[1], "setup")) { - if (argc != 4) - die_with_usage(); - int uid = parseUID(argv[2]); - char *uuid = parseUUID(argv[3]); - setup(uid, uuid); - return 0; - } - if (!strcmp(argv[1], "teardown")) { - if (argc != 4) - die_with_usage(); - int uid = strcmp(argv[2], "*") ? parseUID(argv[2]) : -1; - char *uuid = strcmp(argv[3], "*") ? parseUUID(argv[3]) : "*"; - teardown(uid, uuid); + exec(parseUUID(argv[2]), argc, &argv[3]); return 0; } die_with_usage(); diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js index 3e5ab36..248dd91 100644 --- a/tools/build-lang-image.js +++ b/tools/build-lang-image.js @@ -28,10 +28,7 @@ async function main() { const hash = await hashDockerfile( "lang", { - "riju:runtime": await getLocalImageLabel( - "riju:runtime", - "riju.image-hash" - ), + "riju:base": await getLocalImageLabel("riju:base", "riju.image-hash"), }, { salt: { @@ -52,7 +49,7 @@ async function main() { try { if (debug) { await runCommand( - `docker run -it --rm -e LANG=${lang} -w /tmp/riju-work --network host riju:runtime` + `docker run -it --rm -e LANG=${lang} -w /tmp/riju-work --network host base:runtime` ); } else { await runCommand( diff --git a/tools/hash-dockerfile.js b/tools/hash-dockerfile.js index bef6be8..8aa6dfa 100644 --- a/tools/hash-dockerfile.js +++ b/tools/hash-dockerfile.js @@ -12,7 +12,7 @@ import _ from "lodash"; import { getLocalImageDigest, getLocalImageLabel } from "./docker-util.js"; import { runCommand } from "./util.js"; -// Given a string like "runtime" that identifies the relevant +// Given a string like "base" that identifies the relevant // Dockerfile, read it from disk and parse it into a list of commands. async function parseDockerfile(name) { const contents = await fs.readFile(`docker/${name}/Dockerfile`, "utf-8"); diff --git a/tools/plan-publish.js b/tools/plan-publish.js index fa389dc..635e46e 100644 --- a/tools/plan-publish.js +++ b/tools/plan-publish.js @@ -149,9 +149,7 @@ async function planDebianPackages(opts) { } clauses.push(`make installs L=${lang}`); clauses.push("make test"); - await runCommand( - `make shell I=runtime CMD="${clauses.join(" && ")}"` - ); + await runCommand(`make shell I=base CMD="${clauses.join(" && ")}"`); } await runCommand(`make upload L=${lang} T=${type}`); }, @@ -186,12 +184,12 @@ async function computePlan() { "ubuntu:rolling": await getLocalImageDigest("ubuntu:rolling"), }; const packaging = await planDockerImage("packaging", dependentHashes); - const runtime = await planDockerImage("runtime", dependentHashes); + const base = await planDockerImage("base", dependentHashes); const packages = await planDebianPackages({ - deps: [packaging.id, runtime.id], + deps: [packaging.id, base.id], }); const composite = await planDockerImage("composite", dependentHashes, { - deps: [runtime.id, ...packages.map(({ id }) => id)], + deps: [base.id, ...packages.map(({ id }) => id)], hashOpts: { salt: { packageHashes: packages.map(({ desired }) => desired).sort(), @@ -202,7 +200,7 @@ async function computePlan() { const app = await planDockerImage("app", dependentHashes, { deps: [composite.id, compile.id], }); - return [packaging, runtime, ...packages, composite, compile, app]; + return [packaging, base, ...packages, composite, compile, app]; } function printTable(data, headers) { From 219fbed3421b955a1c7e9d8624db3e8e648c8b12 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 20 Mar 2021 22:15:49 -0700 Subject: [PATCH 007/104] Start work on depgraph runner --- Makefile | 40 ++--- backend/test-runner.js | 32 +++- bin/dep | 5 + lib/hash-test.js | 83 +++++++++ lib/yaml.js | 7 + package.json | 4 +- tools/build-lang-image.js | 4 +- tools/depgraph.js | 304 +++++++++++++++++++++++++++++++++ tools/generate-build-script.js | 2 +- tools/hash-dockerfile.js | 18 ++ yarn.lock | 28 ++- 11 files changed, 489 insertions(+), 38 deletions(-) create mode 100755 bin/dep create mode 100644 lib/hash-test.js create mode 100644 tools/depgraph.js diff --git a/Makefile b/Makefile index caf0d99..51c3d5f 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,9 @@ image: # I= [L=] [NC=1] : Build a Docker image ifeq ($(I),lang) @: $${L} node tools/build-lang-image.js --lang $(L) +else ifeq ($(I),ubuntu) + docker pull ubuntu:rolling + docker tag ubuntu:rolling riju:ubuntu else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else @@ -66,21 +69,23 @@ else LANG_TAG := $(I) endif +IMAGE_HASH := -e RIJU_IMAGE_HASH="$$(docker inspect riju:$(LANG_TAG) | jq '.[0].Config.Labels["riju.image-hash"]' -r)" + shell: # I= [L=] [E=1] [P1|P2=] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) - docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) else ifneq (,$(filter $(I),base lang)) ifeq ($(I),lang) @: $${L} endif - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) else ifeq ($(I),runtime) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) else - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) endif ## This is equivalent to 'make pkg' in a fresh packaging container @@ -194,11 +199,12 @@ dev: # Compile, run, and watch all artifacts and server for development ### Application tools -## L can be a language identifier or a test type (run, repl, lsp, -## format, etc.). Multiple identifiers can be separated by spaces to -## form a conjunction (AND), or by commas to form a disjunction (OR). +## L is a language identifier or a comma-separated list of them, to +## filter tests by language. T is a test type (run, repl, lsp, format, +## etc.) or a set of them to filter tests that way. If both filters +## are provided, then only tests matching both are run. -test: # L= : Run test(s) for language or test category +test: # [L=[,...]] [T=[,...]] : Run test(s) for language or test category node backend/test-runner.js $(L) ## Functions such as 'repl', 'run', 'format', etc. are available in @@ -219,9 +225,6 @@ lsp: # L= : Run LSP REPL for language or custom command line ### Fetch artifacts from registries -pull-base: # Pull latest base image(s) from Docker Hub - docker pull ubuntu:rolling - pull: # I= : Pull last published Riju image from Docker Hub @: $${I} $${DOCKER_REPO} docker pull $(DOCKER_REPO):$(I) @@ -232,12 +235,6 @@ download: # L= T= : Download last published .deb from S3 mkdir -p $(BUILD) aws s3 cp $(S3_DEB) $(BUILD)/$(DEB) --no-sign-request -plan: # Display plan to pull/rebuild outdated or missing artifacts - node tools/plan-publish.js - -sync: # Pull/rebuild outdated or missing artifacts - node tools/plan-publish.js --execute - ### Publish artifacts to registries push: # I= : Push Riju image to Docker Hub @@ -251,11 +248,6 @@ upload: # L= T= : Upload .deb to S3 aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null -## You should probably only run this from CI. - -publish: # Full synchronization and prod deployment - tools/publish.bash - ### Miscellaneous ## Run this every time you update .gitignore or .dockerignore.in. @@ -274,7 +266,7 @@ env: # Run shell with .env file loaded and $PATH fixed tmux: # Start or attach to tmux session MAKELEVEL= tmux attach || MAKELEVEL= tmux new-session -s tmux -usage: + usage: @cat Makefile | \ grep -E '^[^.:[:space:]]+:|[#]##' | \ sed -E 's/:[^#]*#([^:]+)$$/: #:\1/' | \ diff --git a/backend/test-runner.js b/backend/test-runner.js index 86a55bc..2e22a6e 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -6,6 +6,7 @@ import pQueue from "p-queue"; const PQueue = pQueue.default; import stripAnsi from "strip-ansi"; +import { getTestHash } from "../lib/hash-test.js"; import * as api from "./api.js"; import { langsPromise } from "./langs.js"; import { getUUID } from "./util.js"; @@ -623,15 +624,11 @@ async function main() { langs = await langsPromise; let tests = getTestList(); const args = process.argv.slice(2); - for (const arg of args) { - tests = tests.filter( - ({ lang, type }) => - arg - .split(",") - .filter((arg) => - [lang, type].concat(langs[lang].aliases || []).includes(arg) - ).length > 0 - ); + if (process.env.L) { + tests = tests.filter(({ lang }) => process.env.L.split().includes(lang)); + } + if (process.env.T) { + tests = tests.filter(({ type }) => process.env.T.split().includes(type)); } if (tests.length === 0) { console.error("no tests selected"); @@ -732,6 +729,23 @@ async function main() { console.error(` - ${lang}/${type} (${err})`) ); } + const langsValidated = {}; + passed.forEach((_, { lang }) => { + langsValidated[lang] = true; + }); + failed.forEach(({ lang }) => { + langsValidated[lang] = false; + }); + for (const [lang, validated] of Object.entries(langsValidated)) { + if (!validated) { + continue; + } + await fs.mkdir(`build/test-hashes/lang`, { recursive: true }); + await fs.writeFile( + `build/test-hashes/lang/${lang}`, + await getTestHash(lang) + ); + } process.exit(failed.size > 0 ? 1 : 0); } diff --git a/bin/dep b/bin/dep new file mode 100755 index 0000000..689e1e5 --- /dev/null +++ b/bin/dep @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +exec node "$(dirname "$(realpath "$0")")"/../tools/depgraph.js "$@" diff --git a/lib/hash-test.js b/lib/hash-test.js new file mode 100644 index 0000000..ce0b860 --- /dev/null +++ b/lib/hash-test.js @@ -0,0 +1,83 @@ +import crypto from "crypto"; +import { promises as fs } from "fs"; +import path from "path"; + +import { parse } from "@babel/parser"; +import { simple as babelWalk } from "babel-walk"; + +import { readLangConfig } from "./yaml.js"; + +async function getRelativeImports(filename) { + const relativeImports = []; + const program = parse(await fs.readFile(filename, "utf-8"), { + sourceType: "module", + plugins: ["classProperties"], + }); + babelWalk({ + ImportDeclaration: (node) => { + if (node.source.type !== "StringLiteral") { + throw new Error(`unsupported import syntax:`, node); + } + const source = node.source.value; + if (!source.startsWith(".")) { + return; + } + relativeImports.push(source); + }, + })(program); + return relativeImports; +} + +function pathRelativeTo(relativePath, relativeTo) { + return path.join(path.dirname(path.resolve(relativeTo)), relativePath); +} + +async function getTransitiveRelativeImports(filename) { + let queue = [filename]; + const found = new Set(); + while (queue.length > 0) { + const filename = path.resolve(queue.pop()); + if (found.has(filename)) { + continue; + } + found.add(filename); + queue = queue.concat( + (await getRelativeImports(filename)).map((result) => + pathRelativeTo(result, filename) + ) + ); + } + return [...found]; +} + +async function getTestRunnerHash() { + const files = await getTransitiveRelativeImports("backend/test-runner.js"); + files.push("package.json"); + files.push("yarn.lock"); + const hashes = []; + for (const file of files) { + hashes.push( + crypto + .createHash("sha1") + .update(await fs.readFile(file, "utf-8")) + .digest("hex") + ); + } + return crypto.createHash("sha1").update(hashes.join(",")).digest("hex"); +} + +const testRunnerHash = getTestRunnerHash(); + +async function getTestConfigHash(lang) { + const config = Object.assign({}, await readLangConfig(lang)); + delete config["install"]; + delete config["info"]; + return crypto.createHash("sha1").update(JSON.stringify(config)).digest("hex"); +} + +export async function getTestHash(lang) { + return crypto + .createHash("sha1") + .update(`${await testRunnerHash},${await getTestConfigHash(lang)}`) + .digest("hex"); +} diff --git a/lib/yaml.js b/lib/yaml.js index 694af9c..27bda8d 100644 --- a/lib/yaml.js +++ b/lib/yaml.js @@ -114,3 +114,10 @@ export async function readSharedDepConfig(lang) { } return fixupLangConfig(langConfig); } + +// Given a language config JSON, return a list of the Riju shared +// dependency names, or an empty list if none are configured for this +// language. +export async function getSharedDepsForLangConfig(langConfig) { + return (langConfig.install && langConfig.install.riju) || []; +} diff --git a/package.json b/package.json index 273156a..6af7b34 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,14 @@ "type": "module", "dependencies": { "@babel/core": "^7.12.10", + "@babel/parser": "^7.13.11", "@babel/preset-env": "^7.12.11", "@balena/dockerignore": "^1.0.2", "async-lock": "^1.2.6", "babel-loader": "^8.2.2", + "babel-walk": "^3.0.0", "buffer": "^6.0.3", - "commander": "^6.2.1", + "commander": "^7.1.0", "css-loader": "^5.0.1", "debounce": "^1.2.0", "docker-file-parser": "^1.0.5", diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js index 248dd91..ffb99fd 100644 --- a/tools/build-lang-image.js +++ b/tools/build-lang-image.js @@ -4,7 +4,7 @@ import http from "http"; import { Command } from "commander"; import express from "express"; -import { readLangConfig } from "../lib/yaml.js"; +import { getSharedDepsForLangConfig, readLangConfig } from "../lib/yaml.js"; import { getLocalImageLabel } from "./docker-util.js"; import { hashDockerfile } from "./hash-dockerfile.js"; import { getDebHash, runCommand } from "./util.js"; @@ -35,7 +35,7 @@ async function main() { langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`), sharedHashes: ( await Promise.all( - (((await readLangConfig(lang)).install || {}).riju || []).map( + getSharedDepsForLangConfig(await readLangConfig(lang)).map( async (name) => await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`) ) diff --git a/tools/depgraph.js b/tools/depgraph.js new file mode 100644 index 0000000..4126cf0 --- /dev/null +++ b/tools/depgraph.js @@ -0,0 +1,304 @@ +import crypto from "crypto"; +import { promises as fs } from "fs"; +import process from "process"; +import url from "url"; + +import { Command } from "commander"; + +import { getTestHash } from "../lib/hash-test.js"; +import { + getLangs, + getSharedDeps, + getSharedDepsForLangConfig, + readLangConfig, +} from "../lib/yaml.js"; +import { + getDockerRepo, + getLocalImageLabel, + getRemoteImageLabel, +} from "./docker-util.js"; +import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js"; +import { runCommand } from "./util.js"; + +function getS3Bucket() { + if (!process.env.S3_BUCKET) { + throw new Error(`unset environment variable: \$S3_BUCKET`); + } + return process.env.S3_BUCKET; +} + +function getInformationalDependencies() { + return { + s3DebHashes: async () => { + return Object.fromEntries( + JSON.parse( + ( + await runCommand( + `aws s3api list-objects-v2 --bucket riju-debs --prefix hashes`, + { getStdout: true } + ) + ).stdout + ).Contents.map(({ Key: key }) => { + const [_, remoteName, remoteHash] = key.split("/"); + return [remoteName, remoteHash]; + }) + ); + }, + s3TestHashes: async () => { + return Object.fromEntries( + JSON.parse( + ( + await runCommand( + `aws s3api list-objects-v2 --bucket riju-debs --prefix test-hashes/lang`, + { getStdout: true } + ) + ).stdout + ).Contents.map(({ Key: key }) => { + const [_1, _2, remoteName, remoteHash] = key.split("/"); + return [remoteName, remoteHash]; + }) + ); + }, + }; +} + +async function getImageArtifact({ tag, isBaseImage, isLangImage }) { + const DOCKER_REPO = getDockerRepo(); + const name = isLangImage ? "lang" : tag; + let baseImageTags = []; + let dependencies = []; + if (!isBaseImage) { + baseImageTags = [...new Set(await getBaseImages(name))].map((baseImage) => { + if (!baseImage.startsWith("riju:")) { + throw new Error( + `non-Riju base image '${baseImage}' in Dockerfile for ${name} image` + ); + } + return baseImage.replace(/^riju:/, ""); + }); + dependencies = baseImageTags.map( + (baseImageName) => `image:${baseImageTag}` + ); + } + if (isLangImage) { + dependencies.push(`deb:lang-${isLangImage.lang}`); + dependencies.concat( + isLangImage.sharedDeps.map((name) => `deb:shared-${name}`) + ); + } + return { + name: `image:${tag}`, + dependencies: dependencies, + getLocalHash: async () => { + return await getLocalImageLabel(`riju:${tag}`, "riju.image-hash"); + }, + getPublishedHash: async () => { + return await getRemoteImageLabel( + `${DOCKER_REPO}:${tag}`, + "riju.image-hash" + ); + }, + getDesiredHash: async (dependencyHashes) => { + if (isBaseImage) { + return null; + } + const dependentDockerHashes = {}; + for (const baseImageTag of baseImageTag) { + dependentDockerHashes[`riju:${baseImageTag}`] = + dependencyHashes[`image:${baseImageTag}`]; + } + const salt = null; + if (isLangImage) { + salt.langHash = dependencyHashes[`deb:lang-${isLangImage.lang}`]; + salt.sharedHashes = isLangImage.sharedDeps.map( + (name) => dependencyHashes[`deb:shared-${name}`] + ); + } + return await hashDockerfile(name, dependentDockerHashes, { salt }); + }, + buildLocally: async () => { + await runCommand(`make image I=${tag}`); + }, + retrieveFromRegistry: async () => { + await runCommand(`make pull I=${tag}`); + }, + publishToRegistry: async () => { + await runCommand(`make push I=${tag}`); + }, + }; +} + +async function getDebArtifact({ type, lang }) { + return { + name: `deb:${type}-${lang}`, + dependencies: ["image:packaging"], + informationalDependencies: { + getPublishedHash: "s3DebHashes", + }, + getLocalHash: async () => { + try { + await fs.access(debPath); + } catch (err) { + return null; + } + return ( + ( + await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { + getStdout: true, + }) + ).stdout.trim() || null + ); + }, + getPublishedHash: async ({ s3DebHashes }) => { + return s3DebHashes[`riju-${type}-${lang}`] || null; + }, + getDesiredHash: async () => { + let contents = await fs.readFile( + `build/${type}/${lang}/build.bash`, + "utf-8" + ); + contents += + (await getLocalImageLabel("riju:packaging", "riju.image-hash")) + "\n"; + return crypto.createHash("sha1").update(contents).digest("hex"); + }, + buildLocally: async () => { + await runCommand( + `make shell I=packaging CMD="make pkg T=${type} L=${lang}"` + ); + }, + retrieveFromRegistry: async () => { + await runCommand(`make download T=${type} L=${lang}`); + }, + publishToRegistry: async () => { + await runCommand(`make upload T=${type} L=${lang}`); + }, + }; +} + +async function getLanguageTestArtifact({ lang }) { + return { + name: `test:lang-${lang}`, + dependencies: ["image:runtime"], + informationalDependencies: { + getPublishedHash: "s3TestHashes", + retrieveFromRegistry: "s3TestHashes", + }, + getLocalHash: async () => { + const hashPath = `build/test-hashes/lang/${lang}`; + let hash; + try { + return (await fs.readFile(hashPath, "utf-8")).trim(); + } catch (err) { + if (err.code === "ENOENT") { + return null; + } else { + throw err; + } + } + }, + getPublishedHash: async ({ s3TestHashes }) => { + return s3TestHashes[lang]; + }, + getDesiredHash: async () => { + return await getTestHash(lang); + }, + buildLocally: async () => { + await runCommand(`make shell I=runtime CMD="make test L=${lang}"`); + }, + retrieveFromRegistry: async ({ s3TestHashes }) => { + await fs.writeFile( + `build/test-hashes/lang/${lang}`, + s3TestHashes[lang] + "\n" + ); + }, + publishToRegistry: async () => { + const hashPath = `build/test-hashes/lang/${lang}`; + const hash = (await fs.readFile(hashPath, "utf-8")).trim(); + const S3_BUCKET = getS3Bucket(); + await runCommand( + `aws s3 cp ${hashPath} s3://${S3_BUCKET}/test-hashes/lang/${lang}/${hash}` + ); + }, + }; +} + +async function getDeployArtifact(langs) { + return { + name: `deploy:prod`, + dependencies: ["image:app"] + .concat(langs.map((lang) => `image:lang-${lang}`)) + .concat(langs.map((lang) => `test:lang-${lang}`)), + getLocalHash: async () => { + return null; + }, + }; +} + +async function getDepGraph() { + const informationalDependencies = getInformationalDependencies(); + const artifacts = []; + artifacts.push( + await getImageArtifact({ + tag: "ubuntu", + isBaseImage: true, + }) + ); + artifacts.push(await getImageArtifact({ tag: "packaging" })); + artifacts.push(await getImageArtifact({ tag: "base" })); + for (const sharedDep of await getSharedDeps()) { + artifacts.push(await getDebArtifact({ type: "shared", lang: sharedDep })); + } + const langs = await getLangs(); + const langConfigs = Object.fromEntries( + await Promise.all( + langs.map(async (lang) => [lang, await readLangConfig(lang)]) + ) + ); + artifacts.push(await getImageArtifact({ tag: "runtime" })); + for (const lang of langs) { + artifacts.push(await getDebArtifact({ type: "lang", lang: lang })); + artifacts.push( + await getImageArtifact({ + tag: `lang-${lang}`, + isLangImage: { + lang: lang, + sharedDeps: await getSharedDepsForLangConfig(langConfigs[lang]), + }, + }) + ); + artifacts.push(await getLanguageTestArtifact({ lang: lang })); + } + artifacts.push(await getImageArtifact({ tag: "app" })); + artifacts.push(await getDeployArtifact(langs)); + return { informationalDependencies, artifacts }; +} + +async function main() { + const program = new Command(); + program.usage("..."); + program.option("--list", "list available artifacts; ignore other arguments"); + program.option("--publish", "publish artifacts to remote registries"); + program.option("--yes", "execute plan without confirmation"); + program.parse(process.argv); + const { list, publish, yes } = program.opts(); + const depgraph = await getDepGraph(); + if (list) { + for (const { name } of depgraph.artifacts) { + console.log(name); + } + console.error(); + console.error(`${depgraph.artifacts.length} artifacts`); + process.exit(0); + } + if (program.args.length === 0) { + program.help({ error: true }); + } + console.log("doing things now"); +} + +if (process.argv[1] === url.fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index df6dfb6..0120c74 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -254,7 +254,7 @@ Description: The ${name} ${ isShared ? "shared dependency" : "language" } packaged for Riju Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g') -Riju-Script-Hash: \$(sha1sum "\$0" | awk '{ print \$1 }')`; +Riju-Script-Hash: \$((cat "\$0"; echo "\${RIJU_IMAGE_HASH}") | sha1sum - | awk '{ print \$1 }')`; parts.push(`\ install -d "\${pkg}/DEBIAN" cat < "\${pkg}/DEBIAN/control" diff --git a/tools/hash-dockerfile.js b/tools/hash-dockerfile.js index 8aa6dfa..2f84bba 100644 --- a/tools/hash-dockerfile.js +++ b/tools/hash-dockerfile.js @@ -64,6 +64,24 @@ async function listFiles(path) { } } +export async function getBaseImages(name) { + const dockerfile = await parseDockerfile(name); + const baseImages = []; + dockerfile.map(({ name, args, error }) => { + if (error) { + throw error; + if (name === "FROM") { + if (typeof args !== "string") { + throw new Error("got unexpected non-string for FROM args"); + } + const image = args.split(" ")[0]; + baseImages.push(image); + } + } + }); + return baseImages; +} + // Given a Dockerfile name like "packaging", read all the necessary // files from disk and then convert the Dockerfile into a JavaScript // object which includes all relevant build context. The idea is that diff --git a/yarn.lock b/yarn.lock index 21eb2e5..91022fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -251,6 +251,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.11.tgz#9ce3595bcd74bc5c466905e86c535b8b25011e79" integrity sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg== +"@babel/parser@^7.13.11": + version "7.13.11" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.11.tgz#f93ebfc99d21c1772afbbaa153f47e7ce2f50b88" + integrity sha512-PhuoqeHoO9fc4ffMEVk4qb/w/s2iOSWohvbHxLtxui0eBg3Lg5gN1U8wp1V1u61hOWkPQJJyJzGH6Y+grwkq8Q== + "@babel/plugin-proposal-async-generator-functions@^7.12.1": version "7.12.12" resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.12.tgz#04b8f24fd4532008ab4e79f788468fd5a8476566" @@ -814,6 +819,15 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" +"@babel/types@^7.9.6": + version "7.13.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.0.tgz#74424d2816f0171b4100f0ab34e9a374efdf7f80" + integrity sha512-hE+HE8rnG1Z6Wzo+MhaKE5lM5eMx71T4EHJgku2E3xIfaULhDcxiiRxUYgwX8qwP1BBSlag+TdGOt6JAidIZTA== + dependencies: + "@babel/helper-validator-identifier" "^7.12.11" + lodash "^4.17.19" + to-fast-properties "^2.0.0" + "@balena/dockerignore@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@balena/dockerignore/-/dockerignore-1.0.2.tgz#9ffe4726915251e8eb69f44ef3547e0da2c03e0d" @@ -1157,6 +1171,13 @@ babel-plugin-dynamic-import-node@^2.3.3: dependencies: object.assign "^4.1.0" +babel-walk@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/babel-walk/-/babel-walk-3.0.0.tgz#71ea81ea1b8e2e7b37540ec8c899fe49d7fb1441" + integrity sha512-fdRxJkQ9MUSEi4jH2DcV3FAPFktk0wefilxrwNyUuWpoWawQGN7G7cB+fOYTtFfI6XNkFgwqJ/D3G18BoJJ/jg== + dependencies: + "@babel/types" "^7.9.6" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -1535,11 +1556,16 @@ commander@^2.20.0: resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== -commander@^6.2.0, commander@^6.2.1: +commander@^6.2.0: version "6.2.1" resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== +commander@^7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-7.1.0.tgz#f2eaecf131f10e36e07d894698226e36ae0eb5ff" + integrity sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg== + commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" From 4a79b95efce039a109606a4e5c6f17d8e60ed683 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 13:55:33 -0700 Subject: [PATCH 008/104] Tear down old containers properly --- backend/api.js | 2 +- system/src/riju-system-privileged.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/api.js b/backend/api.js index 86cdb0c..5afeaf2 100644 --- a/backend/api.js +++ b/backend/api.js @@ -442,7 +442,7 @@ export class Session { // block signals. Killing the foreground process (i.e. pid1) // should cause the Docker runtime to bring everything else // down in flames. - this.container.proc.kill(); + this.container.proc.stdin.end(); } allSessions.delete(this); this.ws.terminate(); diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 44a0f6b..506d5bc 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -51,7 +51,7 @@ void session(char *uuid, char *lang) char *argv[] = { "docker", "run", - "--rm", + "--rm", "-i", "-e", "HOME=/home/riju", "-e", "HOSTNAME=riju", "-e", "LANG=C.UTF-8", @@ -66,7 +66,7 @@ void session(char *uuid, char *lang) "-e", "USERNAME=riju", "--hostname", "riju", "--name", container, - image, "tail", "-f", "/dev/null", NULL, + image, "cat", NULL, }; execvp(argv[0], argv); die("execvp failed"); From 2e12413d543673113f900cc39a4aa44e068dc6d8 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 14:15:45 -0700 Subject: [PATCH 009/104] Fix popen/pclose usage --- system/src/riju-system-privileged.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 506d5bc..9c0ae9f 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -81,7 +81,7 @@ void wait_alarm(int signum) void wait(char *uuid) { char *cmdline; - if (asprintf(&cmdline, "docker inspect riju-session-%s", uuid) < 0) + if (asprintf(&cmdline, "docker inspect riju-session-%s >/dev/null 2>&1", uuid) < 0) die("asprintf failed"); struct timespec ts; ts.tv_sec = 0; @@ -92,6 +92,10 @@ void wait(char *uuid) FILE *proc = popen(cmdline, "r"); if (proc == NULL) die("popen failed"); + char buf[1024]; + while (fgets(buf, 1024, proc) != NULL); + if (ferror(proc)) + die("fgets failed"); int status = pclose(proc); if (status < 0) die("pclose failed"); From 69f1c2ed58b094ee524a80c808ae9e1157a6e681 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 14:16:23 -0700 Subject: [PATCH 010/104] Remove no longer correct comment --- backend/api.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/backend/api.js b/backend/api.js index 5afeaf2..1f06812 100644 --- a/backend/api.js +++ b/backend/api.js @@ -437,11 +437,6 @@ export class Session { this.log(`Tearing down session`); this.tearingDown = true; if (this.container) { - // SIGTERM should be sufficient as the command running in the - // foreground is just 'tail -f /dev/null' which won't try to - // block signals. Killing the foreground process (i.e. pid1) - // should cause the Docker runtime to bring everything else - // down in flames. this.container.proc.stdin.end(); } allSessions.delete(this); From 224f8f7d9d45e2552c19f9b118a8eacd2ca3dc78 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 14:36:44 -0700 Subject: [PATCH 011/104] Misc fixes for containerization --- backend/api.js | 2 +- docker/base/Dockerfile | 5 ++++- docker/lang/Dockerfile | 6 ------ docker/lang/install.bash | 8 +++++--- tools/build-lang-image.js | 5 +++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/api.js b/backend/api.js index 1f06812..040d391 100644 --- a/backend/api.js +++ b/backend/api.js @@ -15,7 +15,7 @@ const allSessions = new Set(); export class Session { get homedir() { - return `/tmp/riju/${this.uuid}`; + return "/home/riju/src"; } get config() { diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 4defbb1..e6a956e 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -3,7 +3,10 @@ FROM ubuntu:rolling COPY docker/base/install.bash /tmp/ RUN /tmp/install.bash -WORKDIR /src +RUN useradd -p '!' -m -l -s /usr/bin/bash riju +RUN runuser -u riju -- mkdir /home/riju/src +WORKDIR /home/riju/src + COPY docker/shared/my_init /usr/local/sbin/ ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] CMD ["bash"] diff --git a/docker/lang/Dockerfile b/docker/lang/Dockerfile index f38b4f8..0bfaf5f 100644 --- a/docker/lang/Dockerfile +++ b/docker/lang/Dockerfile @@ -5,10 +5,4 @@ ARG LANG COPY docker/lang/install.bash /tmp/ RUN /tmp/install.bash -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] -RUN rm /usr/local/sbin/pid1.bash - -RUN useradd -p '!' -m -l -s /usr/bin/bash riju -WORKDIR /home/riju/src USER riju -CMD ["bash"] diff --git a/docker/lang/install.bash b/docker/lang/install.bash index c0301e7..41f4f04 100755 --- a/docker/lang/install.bash +++ b/docker/lang/install.bash @@ -28,9 +28,11 @@ if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then apt-get update fi -for file in ./riju-shared-*.deb; do - apt-get install -y "${file}" -done +if compgen -G "./riju-shared-*.deb"; then + for file in ./riju-shared-*.deb; do + apt-get install -y "${file}" + done +fi apt-get install -y "./riju-lang-${LANG}.deb" diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js index ffb99fd..d9016aa 100644 --- a/tools/build-lang-image.js +++ b/tools/build-lang-image.js @@ -1,5 +1,6 @@ import { promises as fs } from "fs"; import http from "http"; +import url from "url"; import { Command } from "commander"; import express from "express"; @@ -24,7 +25,7 @@ async function main() { program.requiredOption("--lang ", "language ID"); program.option("--debug", "interactive debugging"); program.parse(process.argv); - const { lang, debug } = program; + const { lang, debug } = program.opts(); const hash = await hashDockerfile( "lang", { @@ -35,7 +36,7 @@ async function main() { langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`), sharedHashes: ( await Promise.all( - getSharedDepsForLangConfig(await readLangConfig(lang)).map( + (await getSharedDepsForLangConfig(await readLangConfig(lang))).map( async (name) => await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`) ) From 1b975de021257dd8b1edb9ea28e6fec562d3d195 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 14:45:13 -0700 Subject: [PATCH 012/104] Get containerized execution working --- backend/api.js | 3 ++- backend/util.js | 4 ++++ system/src/riju-system-privileged.c | 16 ++++++++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/backend/api.js b/backend/api.js index 040d391..9606592 100644 --- a/backend/api.js +++ b/backend/api.js @@ -50,6 +50,7 @@ export class Session { privilegedSession = () => util.privilegedSession(this.context); privilegedWait = () => util.privilegedWait(this.context); privilegedExec = (args) => util.privilegedExec(this.context, args); + privilegedPty = (args) => util.privilegedPty(this.context, args); setup = async () => { try { @@ -324,7 +325,7 @@ export class Session { code += suffix + "\n"; } await this.writeCode(code); - const termArgs = this.privilegedExec(bash(cmdline)); + const termArgs = this.privilegedPty(bash(cmdline)); const term = { pty: pty.spawn(termArgs[0], termArgs.slice(1), { name: "xterm-color", diff --git a/backend/util.js b/backend/util.js index ce29b8a..aebbd1b 100644 --- a/backend/util.js +++ b/backend/util.js @@ -56,6 +56,10 @@ export function privilegedExec({ uuid }, args) { return [rijuSystemPrivileged, "exec", uuid].concat(args); } +export function privilegedPty({ uuid }, args) { + return [rijuSystemPrivileged, "pty", uuid].concat(args); +} + export function bash(cmdline) { if (!cmdline.match(/[;|&(){}=\n]/)) { // Reduce number of subshells we generate, if we're just running a diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 9c0ae9f..afd7210 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,8 @@ void die_with_usage() die("usage:\n" " riju-system-privileged session UUID LANG\n" " riju-system-privileged wait UUID\n" - " riju-system-privileged exec UUID CMDLINE..."); + " riju-system-privileged exec UUID CMDLINE...\n" + " riju-system-privileged pty UUID CMDLINE..."); } char *parseUUID(char *uuid) @@ -107,7 +109,7 @@ void wait(char *uuid) } } -void exec(char *uuid, int argc, char **cmdline) +void exec(char *uuid, int argc, char **cmdline, bool pty) { char *container; if (asprintf(&container, "riju-session-%s", uuid) < 0) @@ -115,7 +117,7 @@ void exec(char *uuid, int argc, char **cmdline) char *argvPrefix[] = { "docker", "exec", - "-it", + pty ? "-it" : "-i", container, }; char **argv = malloc(sizeof(argvPrefix) + (argc + 1) * sizeof(char *)); @@ -152,7 +154,13 @@ int main(int argc, char **argv) if (!strcmp(argv[1], "exec")) { if (argc < 4) die_with_usage(); - exec(parseUUID(argv[2]), argc, &argv[3]); + exec(parseUUID(argv[2]), argc, &argv[3], false); + return 0; + } + if (!strcmp(argv[1], "pty")) { + if (argc < 4) + die_with_usage(); + exec(parseUUID(argv[2]), argc, &argv[3], true); return 0; } die_with_usage(); From 00c3a4117a695f515a4a51b29e4f7dc2ab07d559 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 14:50:45 -0700 Subject: [PATCH 013/104] Finish up initial depgraph, remove dead code --- tools/depgraph.js | 11 ++- tools/deploy.bash | 9 +- tools/plan-publish.js | 186 ------------------------------------------ tools/publish.bash | 39 --------- 4 files changed, 16 insertions(+), 229 deletions(-) delete mode 100755 tools/publish.bash diff --git a/tools/depgraph.js b/tools/depgraph.js index 4126cf0..1a047d3 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -228,8 +228,9 @@ async function getDeployArtifact(langs) { dependencies: ["image:app"] .concat(langs.map((lang) => `image:lang-${lang}`)) .concat(langs.map((lang) => `test:lang-${lang}`)), - getLocalHash: async () => { - return null; + publishOnly: true, + publishToRegistry: async () => { + await runCommand(`tools/deploy.bash`); }, }; } @@ -273,6 +274,10 @@ async function getDepGraph() { return { informationalDependencies, artifacts }; } +async function executeDepGraph({ publish, yes, targets }) { + // +} + async function main() { const program = new Command(); program.usage("..."); @@ -293,7 +298,7 @@ async function main() { if (program.args.length === 0) { program.help({ error: true }); } - console.log("doing things now"); + await executeDepGraph({ publish, yes, targets: program.args }); } if (process.argv[1] === url.fileURLToPath(import.meta.url)) { diff --git a/tools/deploy.bash b/tools/deploy.bash index a72089d..effdb02 100755 --- a/tools/deploy.bash +++ b/tools/deploy.bash @@ -2,6 +2,13 @@ set -euo pipefail +if [[ -z "${DEPLOY_SSH_PRIVATE_KEY:-}" ]]; then + : ${DEPLOY_SSH_PUBLIC_KEY_FILE} + DEPLOY_SSH_PRIVATE_KEY="$(base64 < "${DEPLOY_SSH_PUBLIC_KEY_FILE%.pub}")" +fi + +: ${DOMAIN} + if (( $# != 1 )); then echo "usage: deploy.bash IMAGE" >&2 exit 1 @@ -30,4 +37,4 @@ chmod go-rwx "${tmpdir}/id" ssh -o IdentitiesOnly=yes \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ - -i "${tmpdir}/id" "deploy@${DOMAIN}" "${image}" + -i "${tmpdir}/id" "deploy@${DOMAIN}" diff --git a/tools/plan-publish.js b/tools/plan-publish.js index 635e46e..6ac8f68 100644 --- a/tools/plan-publish.js +++ b/tools/plan-publish.js @@ -17,192 +17,6 @@ import { import { hashDockerfile } from "./hash-dockerfile.js"; import { runCommand } from "./util.js"; -async function planDockerImage(name, dependentHashes, opts) { - const { deps, hashOpts } = opts || {}; - const DOCKER_REPO = getDockerRepo(); - const desired = await hashDockerfile(name, dependentHashes, hashOpts); - const local = await getLocalImageLabel(`riju:${name}`, "riju.image-hash"); - const remote = await getRemoteImageLabel( - `${DOCKER_REPO}:${name}`, - "riju.image-hash" - ); - dependentHashes[`riju:${name}`] = desired; - return { - id: getUUID(), - deps: deps || [], - artifact: "Docker image", - name, - desired, - local, - remote, - download: async () => { - if (name === "app") { - // Magic string parsed by publish.bash - console.log("[for publish script: plan-publish is tagging app image]"); - } - await runCommand(`make pull I=${name}`); - }, - build: async () => { - if (name === "app") { - // Magic string parsed by publish.bash - console.log("[for publish script: plan-publish is tagging app image]"); - } - await runCommand(`make image I=${name}`); - }, - upload: async () => { - if (name === "composite") { - await runCommand(`make shell I=composite CMD="make test"`); - } - await runCommand(`make push I=${name}`); - }, - }; -} - -async function planDebianPackages(opts) { - const { deps } = opts || {}; - const remoteHashes = Object.fromEntries( - JSON.parse( - ( - await runCommand( - `aws s3api list-objects-v2 --bucket riju-debs --prefix hashes`, - { getStdout: true } - ) - ).stdout - ).Contents.map(({ Key: key }) => { - const [_, remoteName, remoteHash] = key.split("/"); - return [remoteName, remoteHash]; - }) - ); - const packages = await getPackages(); - const uuids = Object.fromEntries( - packages.map(({ name }) => [name, getUUID()]) - ); - const langUUIDs = Object.fromEntries( - packages - .filter(({ type }) => type === "lang") - .map(({ lang, name }) => [lang, uuids[name]]) - ); - const sharedUUIDs = Object.fromEntries( - packages - .filter(({ type }) => type === "shared") - .map(({ lang }) => [lang, uuids[lang]]) - ); - const langConfigs = Object.fromEntries( - await Promise.all( - (await getLangs()).map(async (id) => [id, await readLangConfig(id)]) - ) - ); - const plan = await Promise.all( - packages.map(async ({ lang, type, name, buildScriptPath, debPath }) => { - const desired = crypto - .createHash("sha1") - .update(await fs.readFile(buildScriptPath, "utf-8")) - .digest("hex"); - let debExists = true; - try { - await fs.access(debPath); - } catch (err) { - debExists = false; - } - let local = null; - if (debExists) { - local = - ( - await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { - getStdout: true, - }) - ).stdout.trim() || null; - } - const remote = remoteHashes[name] || null; - let sharedDeps = []; - if (type === "lang") { - const cfg = langConfigs[lang]; - sharedDeps = ((cfg.install && cfg.install.riju) || []).map( - (id) => sharedUUIDs[id] - ); - } - return { - id: uuids[name], - deps: [ - ...(deps || []), - ...(type === "config" ? [langUUIDs[lang]] : []), - ...sharedDeps, - ], - artifact: "Debian package", - name, - desired, - local, - remote, - download: async () => { - await runCommand(`make download L=${lang} T=${type}`); - }, - build: async () => { - await runCommand( - `make shell I=packaging CMD="make pkg L=${lang} T=${type}"` - ); - }, - upload: async () => { - if (type === "config") { - const clauses = []; - for (const dep of (langConfigs[lang].install || {}).riju || []) { - clauses.push(`make install T=shared L=${dep}`); - } - clauses.push(`make installs L=${lang}`); - clauses.push("make test"); - await runCommand(`make shell I=base CMD="${clauses.join(" && ")}"`); - } - await runCommand(`make upload L=${lang} T=${type}`); - }, - type, - lang, - }; - }) - ); - const lazilyDownloadedLanguages = new Set(); - for (const { type, lang, desired, local, remote } of plan) { - if (type === "shared") { - continue; - } - // If *not* a shared package, and all we have to do is download - // it, then sort to the end. Unless of course this is the lang - // package, and we need to rebuild the config package, in which - // case the config package (which comes later) will remove that - // lang from the set again. - if (local !== desired && remote === desired) { - lazilyDownloadedLanguages.add(lang); - } else { - lazilyDownloadedLanguages.delete(lang); - } - } - return _.sortBy(plan, ({ type, lang }) => { - return type !== "shared" && lazilyDownloadedLanguages.has(lang); - }); -} - -async function computePlan() { - const dependentHashes = { - "ubuntu:rolling": await getLocalImageDigest("ubuntu:rolling"), - }; - const packaging = await planDockerImage("packaging", dependentHashes); - const base = await planDockerImage("base", dependentHashes); - const packages = await planDebianPackages({ - deps: [packaging.id, base.id], - }); - const composite = await planDockerImage("composite", dependentHashes, { - deps: [base.id, ...packages.map(({ id }) => id)], - hashOpts: { - salt: { - packageHashes: packages.map(({ desired }) => desired).sort(), - }, - }, - }); - const compile = await planDockerImage("compile", dependentHashes); - const app = await planDockerImage("app", dependentHashes, { - deps: [composite.id, compile.id], - }); - return [packaging, base, ...packages, composite, compile, app]; -} - function printTable(data, headers) { const widths = headers.map(({ key, title }) => Math.max(title.length, ...data.map((datum) => datum[key].length)) diff --git a/tools/publish.bash b/tools/publish.bash deleted file mode 100755 index a7eae52..0000000 --- a/tools/publish.bash +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -: ${DOCKER_REPO} -: ${DOMAIN} -: ${S3_BUCKET} - -if [[ -z "${DEPLOY_SSH_PRIVATE_KEY:-}" ]]; then - : ${DEPLOY_SSH_PUBLIC_KEY_FILE} - DEPLOY_SSH_PRIVATE_KEY="$(base64 < "${DEPLOY_SSH_PUBLIC_KEY_FILE%.pub}")" -fi - -tmpdir="$(mktemp -d)" - -function cleanup { - rm -rf "${tmpdir}" -} - -trap cleanup EXIT - -make pull-base all-scripts - -node tools/plan-publish.js --execute --publish --show-all --omit-unneeded-downloads \ - | tee "${tmpdir}/plan-publish.out" - -if ! grep -F "plan-publish is tagging app image" "${tmpdir}/plan-publish.out"; then - echo "publish.bash: no changes to app image, so not deploying" >&2 - exit 0 -fi - -sha="$(git describe --match=always-omit-tag --always --abbrev=40 --dirty)" - -image="${DOCKER_REPO}:app-${sha}" - -docker tag "${DOCKER_REPO}:app" "${image}" -docker push "${image}" - -exec tools/deploy.bash "${image}" From b986cf69af1f8743da326b738f5dd92ada4d0cf9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 18:09:11 -0700 Subject: [PATCH 014/104] Compute transitive dependencies --- docker/app/Dockerfile | 2 +- docker/base/Dockerfile | 2 +- docker/packaging/Dockerfile | 2 +- docker/runtime/Dockerfile | 2 +- tools/depgraph.js | 37 +++++++++++++++++++++++++++++-------- tools/hash-dockerfile.js | 12 ++++++------ 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index cd36ed1..7b8a710 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:rolling AS build +FROM riju:ubuntu AS build COPY docker/app/install-build.bash /tmp/ RUN /tmp/install-build.bash diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index e6a956e..9d23860 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:rolling +FROM riju:ubuntu COPY docker/base/install.bash /tmp/ RUN /tmp/install.bash diff --git a/docker/packaging/Dockerfile b/docker/packaging/Dockerfile index 5ac6598..899afb8 100644 --- a/docker/packaging/Dockerfile +++ b/docker/packaging/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:rolling +FROM riju:ubuntu COPY docker/packaging/install.bash /tmp/ RUN /tmp/install.bash diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile index 62acade..0ac0212 100644 --- a/docker/runtime/Dockerfile +++ b/docker/runtime/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:rolling +FROM riju:ubuntu COPY docker/runtime/install.bash /tmp/ RUN /tmp/install.bash diff --git a/tools/depgraph.js b/tools/depgraph.js index 1a047d3..c9deaa5 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -4,6 +4,7 @@ import process from "process"; import url from "url"; import { Command } from "commander"; +import _ from "lodash"; import { getTestHash } from "../lib/hash-test.js"; import { @@ -76,13 +77,11 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { } return baseImage.replace(/^riju:/, ""); }); - dependencies = baseImageTags.map( - (baseImageName) => `image:${baseImageTag}` - ); + dependencies = baseImageTags.map((baseImageTag) => `image:${baseImageTag}`); } if (isLangImage) { dependencies.push(`deb:lang-${isLangImage.lang}`); - dependencies.concat( + dependencies = dependencies.concat( isLangImage.sharedDeps.map((name) => `deb:shared-${name}`) ); } @@ -178,7 +177,7 @@ async function getDebArtifact({ type, lang }) { async function getLanguageTestArtifact({ lang }) { return { name: `test:lang-${lang}`, - dependencies: ["image:runtime"], + dependencies: ["image:runtime", `image:lang-${lang}`], informationalDependencies: { getPublishedHash: "s3TestHashes", retrieveFromRegistry: "s3TestHashes", @@ -274,8 +273,30 @@ async function getDepGraph() { return { informationalDependencies, artifacts }; } -async function executeDepGraph({ publish, yes, targets }) { - // +function getTransitiveDependencies({ artifacts, targets }) { + let queue = targets; + let found = new Set(); + while (queue.length > 0) { + const name = queue.pop(); + if (found.has(name)) { + continue; + } + if (!artifacts[name]) { + throw new Error(`no such artifact: ${name}`); + } + queue = queue.concat(artifacts[name].dependencies); + found.add(name); + } + return _.sortBy([...found], (name) => Object.keys(artifacts).indexOf(name)); +} + +async function executeDepGraph({ depgraph, publish, yes, targets }) { + const artifacts = {}; + for (const artifact of depgraph.artifacts) { + artifacts[artifact.name] = artifact; + } + const transitiveTargets = getTransitiveDependencies({ artifacts, targets }); + console.log(transitiveTargets); } async function main() { @@ -298,7 +319,7 @@ async function main() { if (program.args.length === 0) { program.help({ error: true }); } - await executeDepGraph({ publish, yes, targets: program.args }); + await executeDepGraph({ depgraph, publish, yes, targets: program.args }); } if (process.argv[1] === url.fileURLToPath(import.meta.url)) { diff --git a/tools/hash-dockerfile.js b/tools/hash-dockerfile.js index 2f84bba..18b1659 100644 --- a/tools/hash-dockerfile.js +++ b/tools/hash-dockerfile.js @@ -70,13 +70,13 @@ export async function getBaseImages(name) { dockerfile.map(({ name, args, error }) => { if (error) { throw error; - if (name === "FROM") { - if (typeof args !== "string") { - throw new Error("got unexpected non-string for FROM args"); - } - const image = args.split(" ")[0]; - baseImages.push(image); + } + if (name === "FROM") { + if (typeof args !== "string") { + throw new Error("got unexpected non-string for FROM args"); } + const image = args.split(" ")[0]; + baseImages.push(image); } }); return baseImages; From 1672e4296053655a626c68a6c909c29bbd9f2bd5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 21 Mar 2021 21:53:22 -0700 Subject: [PATCH 015/104] Bugfixes for hashing --- backend/test-runner.js | 2 +- lib/hash-test.js | 6 ++-- tools/depgraph.js | 65 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/backend/test-runner.js b/backend/test-runner.js index 2e22a6e..c5be3d6 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -743,7 +743,7 @@ async function main() { await fs.mkdir(`build/test-hashes/lang`, { recursive: true }); await fs.writeFile( `build/test-hashes/lang/${lang}`, - await getTestHash(lang) + await getTestHash(lang, process.env.RIJU_IMAGE_HASH) ); } process.exit(failed.size > 0 ? 1 : 0); diff --git a/lib/hash-test.js b/lib/hash-test.js index ce0b860..dd7a52b 100644 --- a/lib/hash-test.js +++ b/lib/hash-test.js @@ -75,9 +75,11 @@ async function getTestConfigHash(lang) { return crypto.createHash("sha1").update(JSON.stringify(config)).digest("hex"); } -export async function getTestHash(lang) { +export async function getTestHash(lang, imageHash) { return crypto .createHash("sha1") - .update(`${await testRunnerHash},${await getTestConfigHash(lang)}`) + .update( + `${await testRunnerHash},${await getTestConfigHash(lang)},${imageHash}` + ) .digest("hex"); } diff --git a/tools/depgraph.js b/tools/depgraph.js index c9deaa5..94ebef3 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -102,16 +102,18 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { return null; } const dependentDockerHashes = {}; - for (const baseImageTag of baseImageTag) { + for (const baseImageTag of baseImageTags) { dependentDockerHashes[`riju:${baseImageTag}`] = dependencyHashes[`image:${baseImageTag}`]; } - const salt = null; + let salt = null; if (isLangImage) { - salt.langHash = dependencyHashes[`deb:lang-${isLangImage.lang}`]; - salt.sharedHashes = isLangImage.sharedDeps.map( - (name) => dependencyHashes[`deb:shared-${name}`] - ); + salt = { + langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`], + sharedHashes: isLangImage.sharedDeps.map( + (name) => dependencyHashes[`deb:shared-${name}`] + ), + }; } return await hashDockerfile(name, dependentDockerHashes, { salt }); }, @@ -151,13 +153,12 @@ async function getDebArtifact({ type, lang }) { getPublishedHash: async ({ s3DebHashes }) => { return s3DebHashes[`riju-${type}-${lang}`] || null; }, - getDesiredHash: async () => { + getDesiredHash: async (dependencyHashes) => { let contents = await fs.readFile( `build/${type}/${lang}/build.bash`, "utf-8" ); - contents += - (await getLocalImageLabel("riju:packaging", "riju.image-hash")) + "\n"; + contents += dependencyHashes["image:packaging"] + "\n"; return crypto.createHash("sha1").update(contents).digest("hex"); }, buildLocally: async () => { @@ -198,8 +199,8 @@ async function getLanguageTestArtifact({ lang }) { getPublishedHash: async ({ s3TestHashes }) => { return s3TestHashes[lang]; }, - getDesiredHash: async () => { - return await getTestHash(lang); + getDesiredHash: async (dependencyHashes) => { + return await getTestHash(lang, dependencyHashes["image:runtime"]); }, buildLocally: async () => { await runCommand(`make shell I=runtime CMD="make test L=${lang}"`); @@ -296,7 +297,47 @@ async function executeDepGraph({ depgraph, publish, yes, targets }) { artifacts[artifact.name] = artifact; } const transitiveTargets = getTransitiveDependencies({ artifacts, targets }); - console.log(transitiveTargets); + const requiredInfo = new Set(); + for (const target of transitiveTargets) { + for (const name of Object.values( + artifacts[target].informationalDependencies || {} + )) { + requiredInfo.add(name); + } + } + const info = {}; + await Promise.all( + [...requiredInfo].map(async (name) => { + info[name] = await depgraph.informationalDependencies[name](); + }) + ); + const hashes = { + local: {}, + published: {}, + desired: {}, + }; + await Promise.all( + transitiveTargets.map(async (target) => { + const { + publishOnly, + getLocalHash, + getPublishedHash, + getDesiredHash, + } = artifacts[target]; + await Promise.all([ + getLocalHash(info).then((hash) => { + hashes.local[target] = hash; + }), + getPublishedHash(info).then((hash) => { + hashes.published[target] = hash; + }), + getDesiredHash(info).then((hash) => { + hashes.desired[target] = hash; + }), + ]); + }) + ); + console.log(hashes); } async function main() { From e1ed1452ec5028c5115767a1a372663b0fba5a75 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 22 Mar 2021 16:26:32 -0700 Subject: [PATCH 016/104] Compute dependent artifact hashes --- tools/depgraph.js | 63 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 94ebef3..99897d5 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -291,11 +291,20 @@ function getTransitiveDependencies({ artifacts, targets }) { return _.sortBy([...found], (name) => Object.keys(artifacts).indexOf(name)); } -async function executeDepGraph({ depgraph, publish, yes, targets }) { +async function executeDepGraph({ depgraph, manual, publish, yes, targets }) { const artifacts = {}; for (const artifact of depgraph.artifacts) { artifacts[artifact.name] = artifact; } + if (manual) { + for (const target of targets) { + if (artifacts[target].dependencies.length > 0) { + throw new Error( + `cannot build target ${target} with --manual as it is not a leaf artifact` + ); + } + } + } const transitiveTargets = getTransitiveDependencies({ artifacts, targets }); const requiredInfo = new Set(); for (const target of transitiveTargets) { @@ -316,6 +325,41 @@ async function executeDepGraph({ depgraph, publish, yes, targets }) { published: {}, desired: {}, }; + const promises = { + local: {}, + published: {}, + desired: {}, + }; + for (const target of transitiveTargets) { + promises.local[target] = artifacts[target].getLocalHash(info); + promises.published[target] = artifacts[target].getPublishedHash(info); + promises.desired[target] = (async () => { + if (manual) { + return null; + } + const dependencyHashes = {}; + for (const dependency of artifacts[target].dependencies) { + dependencyHashes[dependency] = await promises.desired[dependency]; + } + let hash = await artifacts[target].getDesiredHash(dependencyHashes); + if (hash !== null) { + return hash; + } + // If hash is missing, cast about in blind panic for another + // possible way to compute it. + hash = await promises.published[target]; + if (hash !== null) { + return hash; + } + hash = await promises.local[target]; + if (hash !== null) { + return hash; + } + throw new Error( + `artifact must be built manually: dep ${target} --manual [--publish]` + ); + })(); + } await Promise.all( transitiveTargets.map(async (target) => { const { @@ -325,13 +369,13 @@ async function executeDepGraph({ depgraph, publish, yes, targets }) { getDesiredHash, } = artifacts[target]; await Promise.all([ - getLocalHash(info).then((hash) => { + promises.local[target].then((hash) => { hashes.local[target] = hash; }), - getPublishedHash(info).then((hash) => { + promises.published[target].then((hash) => { hashes.published[target] = hash; }), - getDesiredHash(info).then((hash) => { + promises.desired[target].then((hash) => { hashes.desired[target] = hash; }), ]); @@ -344,10 +388,11 @@ async function main() { const program = new Command(); program.usage("..."); program.option("--list", "list available artifacts; ignore other arguments"); + program.option("--manual", "operate manually on leaf artifacts"); program.option("--publish", "publish artifacts to remote registries"); program.option("--yes", "execute plan without confirmation"); program.parse(process.argv); - const { list, publish, yes } = program.opts(); + const { list, manual, publish, yes } = program.opts(); const depgraph = await getDepGraph(); if (list) { for (const { name } of depgraph.artifacts) { @@ -360,7 +405,13 @@ async function main() { if (program.args.length === 0) { program.help({ error: true }); } - await executeDepGraph({ depgraph, publish, yes, targets: program.args }); + await executeDepGraph({ + depgraph, + manual, + publish, + yes, + targets: program.args, + }); } if (process.argv[1] === url.fileURLToPath(import.meta.url)) { From 17d016761146f1fe29be0ee7a32275bb88f3c2f9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 23 Mar 2021 19:14:01 -0700 Subject: [PATCH 017/104] Slightly more correct but probably same effect --- tools/depgraph.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 99897d5..85ba6b7 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -335,7 +335,8 @@ async function executeDepGraph({ depgraph, manual, publish, yes, targets }) { promises.published[target] = artifacts[target].getPublishedHash(info); promises.desired[target] = (async () => { if (manual) { - return null; + // Artifact has no dependencies in this case. + return await artifacts[target].getDesiredHash({}); } const dependencyHashes = {}; for (const dependency of artifacts[target].dependencies) { From 76a2a2d1ee629e390e687e3c46806ce24ecb83f6 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 26 Mar 2021 20:31:46 -0700 Subject: [PATCH 018/104] Misc refactoring --- tools/depgraph.js | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 85ba6b7..0e35bea 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -291,7 +291,14 @@ function getTransitiveDependencies({ artifacts, targets }) { return _.sortBy([...found], (name) => Object.keys(artifacts).indexOf(name)); } -async function executeDepGraph({ depgraph, manual, publish, yes, targets }) { +async function executeDepGraph({ + depgraph, + manual, + holdManual, + publish, + yes, + targets, +}) { const artifacts = {}; for (const artifact of depgraph.artifacts) { artifacts[artifact.name] = artifact; @@ -334,30 +341,31 @@ async function executeDepGraph({ depgraph, manual, publish, yes, targets }) { promises.local[target] = artifacts[target].getLocalHash(info); promises.published[target] = artifacts[target].getPublishedHash(info); promises.desired[target] = (async () => { - if (manual) { - // Artifact has no dependencies in this case. - return await artifacts[target].getDesiredHash({}); - } const dependencyHashes = {}; for (const dependency of artifacts[target].dependencies) { dependencyHashes[dependency] = await promises.desired[dependency]; + if (!dependencyHashes[dependency]) { + throw new Error( + `manual dependency must be built explicitly: dep ${target} --manual [--publish]` + ); + } } let hash = await artifacts[target].getDesiredHash(dependencyHashes); - if (hash !== null) { + if (hash || manual) { return hash; } - // If hash is missing, cast about in blind panic for another - // possible way to compute it. - hash = await promises.published[target]; - if (hash !== null) { - return hash; + const promiseSets = [promises.published, promises.local]; + if (holdLocal) { + sets.reverse(); } - hash = await promises.local[target]; - if (hash !== null) { - return hash; + for (const promiseSet of promiseSets) { + const hash = await promiseSet[target]; + if (hash) { + return hash; + } } throw new Error( - `artifact must be built manually: dep ${target} --manual [--publish]` + `manual artifact must be built explicitly: dep ${target} --manual [--publish]` ); })(); } @@ -389,11 +397,12 @@ async function main() { const program = new Command(); program.usage("..."); program.option("--list", "list available artifacts; ignore other arguments"); - program.option("--manual", "operate manually on leaf artifacts"); + program.option("--manual", "operate explicitly on manual artifacts"); + program.option("--hold-manual", "prefer local versions of manual artifacts"); program.option("--publish", "publish artifacts to remote registries"); program.option("--yes", "execute plan without confirmation"); program.parse(process.argv); - const { list, manual, publish, yes } = program.opts(); + const { list, manual, holdManual, publish, yes } = program.opts(); const depgraph = await getDepGraph(); if (list) { for (const { name } of depgraph.artifacts) { @@ -409,6 +418,7 @@ async function main() { await executeDepGraph({ depgraph, manual, + holdManual, publish, yes, targets: program.args, From 9bd99c027df9acfcadb64ff12619b77a6255efc4 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 26 Mar 2021 21:14:19 -0700 Subject: [PATCH 019/104] Misc bugfixes --- Makefile | 2 +- tools/depgraph.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 51c3d5f..e0cf23e 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ ifeq ($(I),lang) node tools/build-lang-image.js --lang $(L) else ifeq ($(I),ubuntu) docker pull ubuntu:rolling - docker tag ubuntu:rolling riju:ubuntu + hash="$$(docker inspect ubuntu:rolling | jq '.[0].Id' -r | sha1sum | awk '{ print $1 }')"; echo "FROM ubuntu:rolling" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) - else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else diff --git a/tools/depgraph.js b/tools/depgraph.js index 0e35bea..6adc92e 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -38,7 +38,7 @@ function getInformationalDependencies() { `aws s3api list-objects-v2 --bucket riju-debs --prefix hashes`, { getStdout: true } ) - ).stdout + ).stdout || '{"Contents": []}' ).Contents.map(({ Key: key }) => { const [_, remoteName, remoteHash] = key.split("/"); return [remoteName, remoteHash]; @@ -53,7 +53,7 @@ function getInformationalDependencies() { `aws s3api list-objects-v2 --bucket riju-debs --prefix test-hashes/lang`, { getStdout: true } ) - ).stdout + ).stdout || '{"Contents": []}' ).Contents.map(({ Key: key }) => { const [_1, _2, remoteName, remoteHash] = key.split("/"); return [remoteName, remoteHash]; @@ -355,7 +355,7 @@ async function executeDepGraph({ return hash; } const promiseSets = [promises.published, promises.local]; - if (holdLocal) { + if (holdManual) { sets.reverse(); } for (const promiseSet of promiseSets) { From 5f523f81426ec0e88177c05a5dbe4acb3508954a Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 26 Mar 2021 21:30:50 -0700 Subject: [PATCH 020/104] More bugfixes --- Makefile | 2 +- tools/depgraph.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e0cf23e..254ef11 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ ifeq ($(I),lang) node tools/build-lang-image.js --lang $(L) else ifeq ($(I),ubuntu) docker pull ubuntu:rolling - hash="$$(docker inspect ubuntu:rolling | jq '.[0].Id' -r | sha1sum | awk '{ print $1 }')"; echo "FROM ubuntu:rolling" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) - + hash="$$(docker inspect ubuntu:rolling | jq '.[0].Id' -r | sha1sum | awk '{ print $$1 }')"; echo "FROM ubuntu:rolling" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) - else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else diff --git a/tools/depgraph.js b/tools/depgraph.js index 6adc92e..a24f9fc 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -197,7 +197,7 @@ async function getLanguageTestArtifact({ lang }) { } }, getPublishedHash: async ({ s3TestHashes }) => { - return s3TestHashes[lang]; + return s3TestHashes[lang] || null; }, getDesiredHash: async (dependencyHashes) => { return await getTestHash(lang, dependencyHashes["image:runtime"]); @@ -356,7 +356,7 @@ async function executeDepGraph({ } const promiseSets = [promises.published, promises.local]; if (holdManual) { - sets.reverse(); + promiseSets.reverse(); } for (const promiseSet of promiseSets) { const hash = await promiseSet[target]; From 5fc14b1403176eae216d62bd8ac0bc7b2bc74acf Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 27 Mar 2021 10:00:26 -0700 Subject: [PATCH 021/104] Compute all hashes correctly --- tools/depgraph.js | 78 +++++++++++-------- tools/docker-util.js | 38 +++++----- tools/plan-publish.js | 169 ------------------------------------------ 3 files changed, 65 insertions(+), 220 deletions(-) delete mode 100644 tools/plan-publish.js diff --git a/tools/depgraph.js b/tools/depgraph.js index a24f9fc..34bf579 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -17,6 +17,7 @@ import { getDockerRepo, getLocalImageLabel, getRemoteImageLabel, + getRemoteRepositoryTags, } from "./docker-util.js"; import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js"; import { runCommand } from "./util.js"; @@ -60,6 +61,9 @@ function getInformationalDependencies() { }) ); }, + dockerRepoTags: async () => { + return await getRemoteRepositoryTags(getDockerRepo()); + }, }; } @@ -88,13 +92,20 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { return { name: `image:${tag}`, dependencies: dependencies, + informationalDependencies: { + getPublishedHash: "dockerRepoTags", + }, getLocalHash: async () => { return await getLocalImageLabel(`riju:${tag}`, "riju.image-hash"); }, - getPublishedHash: async () => { + getPublishedHash: async ({ dockerRepoTags }) => { + if (!dockerRepoTags.includes(tag)) { + return null; + } return await getRemoteImageLabel( `${DOCKER_REPO}:${tag}`, - "riju.image-hash" + "riju.image-hash", + dockerRepoTags ); }, getDesiredHash: async (dependencyHashes) => { @@ -137,6 +148,7 @@ async function getDebArtifact({ type, lang }) { getPublishedHash: "s3DebHashes", }, getLocalHash: async () => { + const debPath = `build/${type}/${lang}/riju-${type}-${lang}.deb`; try { await fs.access(debPath); } catch (err) { @@ -228,7 +240,7 @@ async function getDeployArtifact(langs) { dependencies: ["image:app"] .concat(langs.map((lang) => `image:lang-${lang}`)) .concat(langs.map((lang) => `test:lang-${lang}`)), - publishOnly: true, + publishTarget: true, publishToRegistry: async () => { await runCommand(`tools/deploy.bash`); }, @@ -338,36 +350,42 @@ async function executeDepGraph({ desired: {}, }; for (const target of transitiveTargets) { - promises.local[target] = artifacts[target].getLocalHash(info); - promises.published[target] = artifacts[target].getPublishedHash(info); - promises.desired[target] = (async () => { - const dependencyHashes = {}; - for (const dependency of artifacts[target].dependencies) { - dependencyHashes[dependency] = await promises.desired[dependency]; - if (!dependencyHashes[dependency]) { - throw new Error( - `manual dependency must be built explicitly: dep ${target} --manual [--publish]` - ); + if (artifacts[target].publishTarget) { + promises.local[target] = Promise.resolve(null); + promises.published[target] = Promise.resolve(null); + promises.desired[target] = Promise.resolve(null); + } else { + promises.local[target] = artifacts[target].getLocalHash(info); + promises.published[target] = artifacts[target].getPublishedHash(info); + promises.desired[target] = (async () => { + const dependencyHashes = {}; + for (const dependency of artifacts[target].dependencies) { + dependencyHashes[dependency] = await promises.desired[dependency]; + if (!dependencyHashes[dependency]) { + throw new Error( + `manual dependency must be built explicitly: dep ${target} --manual [--publish]` + ); + } } - } - let hash = await artifacts[target].getDesiredHash(dependencyHashes); - if (hash || manual) { - return hash; - } - const promiseSets = [promises.published, promises.local]; - if (holdManual) { - promiseSets.reverse(); - } - for (const promiseSet of promiseSets) { - const hash = await promiseSet[target]; - if (hash) { + let hash = await artifacts[target].getDesiredHash(dependencyHashes); + if (hash || manual) { return hash; } - } - throw new Error( - `manual artifact must be built explicitly: dep ${target} --manual [--publish]` - ); - })(); + const promiseSets = [promises.published, promises.local]; + if (holdManual) { + promiseSets.reverse(); + } + for (const promiseSet of promiseSets) { + const hash = await promiseSet[target]; + if (hash) { + return hash; + } + } + throw new Error( + `manual artifact must be built explicitly: dep ${target} --manual [--publish]` + ); + })(); + } } await Promise.all( transitiveTargets.map(async (target) => { diff --git a/tools/docker-util.js b/tools/docker-util.js index ef310a9..7121d46 100644 --- a/tools/docker-util.js +++ b/tools/docker-util.js @@ -24,26 +24,29 @@ export async function getLocalImageLabel(image, label) { await runCommand(`docker inspect "${image}"`, { getStdout: true }) ).stdout; } catch (err) { - if ( - (await runCommand(`docker images -q "${image}"`, { getStdout: true })) - .stdout - ) { - // The image exists locally, something unexpected must have - // happened in docker inspect. - throw err; - } else { - // The image doesn't exist locally, that must be why docker - // inspect didn't work. - return null; - } + return null; } const labels = JSON.parse(output)[0].Config.Labels; return (labels && labels[label]) || null; } +// Return the list of tags in a remote Docker repository. +export async function getRemoteRepositoryTags(repo) { + return JSON.parse( + ( + await runCommand(`skopeo list-tags "docker://${repo}"`, { + getStdout: true, + }) + ).stdout + ).Tags; +} + // Return the value of a label on a Docker image that is on a remote -// registry. If the image or label doesn't exist, return null. -export async function getRemoteImageLabel(image, label) { +// registry. If the image or label doesn't exist, return null. You +// have to pass in a list of tags on the remote repository (see +// getRemoteRepositoryTags) so that we can distinguish between missing +// images and network errors. +export async function getRemoteImageLabel(image, label, tags) { const [repo, tag] = image.split(":"); let output; try { @@ -53,13 +56,6 @@ export async function getRemoteImageLabel(image, label) { }) ).stdout; } catch (err) { - const tags = JSON.parse( - ( - await runCommand(`skopeo list-tags "docker://${repo}"`, { - getStdout: true, - }) - ).stdout - ).Tags; if (tags.includes(tag)) { // Tag exists, something unexpected must have gone wrong when // running skopeo inspect. diff --git a/tools/plan-publish.js b/tools/plan-publish.js deleted file mode 100644 index 6ac8f68..0000000 --- a/tools/plan-publish.js +++ /dev/null @@ -1,169 +0,0 @@ -import crypto from "crypto"; -import { promises as fs } from "fs"; -import process from "process"; -import url from "url"; - -import { Command } from "commander"; -import _ from "lodash"; -import { v4 as getUUID } from "uuid"; - -import { getLangs, getPackages, readLangConfig } from "../lib/yaml.js"; -import { - getLocalImageDigest, - getLocalImageLabel, - getRemoteImageLabel, - getDockerRepo, -} from "./docker-util.js"; -import { hashDockerfile } from "./hash-dockerfile.js"; -import { runCommand } from "./util.js"; - -function printTable(data, headers) { - const widths = headers.map(({ key, title }) => - Math.max(title.length, ...data.map((datum) => datum[key].length)) - ); - [ - headers.map(({ title }) => title.toUpperCase()), - widths.map((width) => "-".repeat(width)), - ...data.map((datum) => headers.map(({ key }) => datum[key])), - ].map((values) => - console.log( - values.map((value, idx) => value.padEnd(widths[idx])).join(" ") - ) - ); -} - -// Parse command-line arguments, run main functionality, and exit. -async function main() { - const program = new Command(); - program.option("--execute", "pull and build artifacts locally"); - program.option( - "--publish", - "with --execute, also deploy newly built artifacts" - ); - program.option("--show-all", "show also unchanged artifacts"); - program.option( - "--omit-unneeded-downloads", - "don't download artifacts unless needed for dependent builds" - ); - program.parse(process.argv); - let plan = await computePlan(); - let tableData = plan.map( - ({ - id, - deps, - artifact, - name, - desired, - local, - remote, - download, - build, - upload, - }) => { - let action, details, func, couldPrune, noop; - if (remote === desired && local === desired) { - action = "(no action)"; - details = desired; - func = () => {}; - couldPrune = true; - noop = true; - } else if (remote === desired && local !== desired) { - action = "download remote"; - details = `${local} => ${desired}`; - func = download; - couldPrune = true; - noop = false; - } else if (local === desired && remote !== desired) { - action = "publish local"; - details = `${remote} => ${desired}`; - func = async () => { - if (program.publish) { - await upload(); - } - }; - couldPrune = false; - noop = false; - } else { - action = "rebuild and publish"; - if (local === remote) { - details = `${local} => ${desired}`; - } else { - details = `${local} (local), ${remote} (remote) => ${desired}`; - } - func = async () => { - await build(); - if (program.publish) { - await upload(); - } - }; - couldPrune = false; - noop = false; - } - return { - id, - deps, - couldPrune, - artifact, - name, - action, - details, - func, - noop, - }; - } - ); - if (program.omitUnneededDownloads) { - for (const datum of [...tableData].reverse()) { - if ( - datum.couldPrune && - _.every( - tableData, - (otherDatum) => - otherDatum.couldPrune || !otherDatum.deps.includes(datum.id) - ) - ) { - datum.pruned = true; - if (!datum.noop) { - datum.action += " [skipping]"; - datum.noop = true; - } - } - } - } - console.log(); - const filteredTableData = tableData.filter(({ noop }) => !noop); - if (filteredTableData.length === 0) { - console.log(`*** NO ACTION REQUIRED FOR ${plan.length} ARTIFACTS ***`); - } else { - console.log( - `*** ACTION REQUIRED FOR ${filteredTableData.length} of ${plan.length} ARTIFACTS ***` - ); - } - console.log(); - if (!program.showAll) { - tableData = filteredTableData; - } - if (tableData.length > 0) { - printTable(tableData, [ - { key: "artifact", title: "Type" }, - { key: "name", title: "Name" }, - { key: "action", title: "Action" }, - { key: "details", title: "Details" }, - ]); - console.log(); - } - tableData = filteredTableData; - if (program.execute) { - for (const { func } of tableData) { - await func(); - } - } - process.exit(0); -} - -if (process.argv[1] === url.fileURLToPath(import.meta.url)) { - main().catch((err) => { - console.error(err); - process.exit(1); - }); -} From d5c9e97e1072bcd168e58101146aeb157388c439 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 27 Mar 2021 16:03:16 -0700 Subject: [PATCH 022/104] Plan and apply --- tools/depgraph.js | 211 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 195 insertions(+), 16 deletions(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 34bf579..526fa3a 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -1,6 +1,7 @@ import crypto from "crypto"; import { promises as fs } from "fs"; import process from "process"; +import readline from "readline"; import url from "url"; import { Command } from "commander"; @@ -287,7 +288,7 @@ async function getDepGraph() { } function getTransitiveDependencies({ artifacts, targets }) { - let queue = targets; + let queue = [...targets]; let found = new Set(); while (queue.length > 0) { const name = queue.pop(); @@ -303,26 +304,56 @@ function getTransitiveDependencies({ artifacts, targets }) { return _.sortBy([...found], (name) => Object.keys(artifacts).indexOf(name)); } +function printTable(data, headers) { + const widths = headers.map(({ key, title }) => + Math.max(title.length, ...data.map((datum) => datum[key].length)) + ); + [ + headers.map(({ title }) => title.toUpperCase()), + widths.map((width) => "-".repeat(width)), + ...data.map((datum) => headers.map(({ key }) => datum[key])), + ].map((values) => + console.log( + values.map((value, idx) => value.padEnd(widths[idx])).join(" ") + ) + ); +} + +async function getUserInput(prompt) { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + prompt: prompt, + }); + rl.prompt(); + const resp = await new Promise((resolve) => { + rl.on("line", (line) => { + resolve(line); + }); + }); + rl.close(); + return resp; +} + async function executeDepGraph({ depgraph, manual, holdManual, + all, publish, yes, targets, }) { const artifacts = {}; for (const artifact of depgraph.artifacts) { - artifacts[artifact.name] = artifact; - } - if (manual) { - for (const target of targets) { - if (artifacts[target].dependencies.length > 0) { + for (const dep of artifact.dependencies) { + if (!artifacts[dep]) { throw new Error( - `cannot build target ${target} with --manual as it is not a leaf artifact` + `artifact ${artifact.name} appears before dependency ${dep} in depgraph` ); } } + artifacts[artifact.name] = artifact; } const transitiveTargets = getTransitiveDependencies({ artifacts, targets }); const requiredInfo = new Set(); @@ -363,7 +394,7 @@ async function executeDepGraph({ dependencyHashes[dependency] = await promises.desired[dependency]; if (!dependencyHashes[dependency]) { throw new Error( - `manual dependency must be built explicitly: dep ${target} --manual [--publish]` + `manual dependency must be built explicitly: dep ${dependency} --manual [--publish]` ); } } @@ -389,12 +420,9 @@ async function executeDepGraph({ } await Promise.all( transitiveTargets.map(async (target) => { - const { - publishOnly, - getLocalHash, - getPublishedHash, - getDesiredHash, - } = artifacts[target]; + const { getLocalHash, getPublishedHash, getDesiredHash } = artifacts[ + target + ]; await Promise.all([ promises.local[target].then((hash) => { hashes.local[target] = hash; @@ -408,7 +436,156 @@ async function executeDepGraph({ ]); }) ); - console.log(hashes); + const statuses = {}; + for (const name in artifacts) { + if (!hashes.desired[name]) { + statuses[name] = "buildLocally"; + } else if ( + hashes.published[name] === hashes.desired[name] && + hashes.local[name] === hashes.desired[name] + ) { + statuses[name] = null; + } else if ( + hashes.local[name] === hashes.desired[name] && + hashes.published[name] !== hashes.desired[name] + ) { + statuses[name] = "publishToRegistry"; + } else if ( + hashes.published[name] === hashes.desired[name] && + hashes.local[name] !== hashes.desired[name] + ) { + statuses[name] = "downloadFromRegistry"; + } else { + statuses[name] = "buildLocally"; + } + } + let priorityTargets; + if (all) { + priorityTargets = transitiveTargets; + } else { + const queue = [...targets]; + const found = new Set(); + while (queue.length > 0) { + const name = queue.pop(); + if (found.has(name)) { + continue; + } + for (const dep of artifacts[name].dependencies) { + if (statuses[dep] === "buildLocally") { + queue.push(dep); + } + } + found.add(name); + } + priorityTargets = [...found]; + } + priorityTargets = _.sortBy(priorityTargets, (name) => + Object.keys(artifacts).indexOf(name) + ); + const plan = []; + const seen = new Set(); + for (const target of priorityTargets) { + for (const dep of artifacts[target].dependencies) { + if (artifacts[target].publishTarget) { + if (statuses === "publishToRegistry") { + plan.push({ + artifact: dep, + action: "publishToRegistry", + }); + } + } else { + if (statuses === "downloadFromRegistry") { + plan.push({ + artifact: dep, + action: "downloadFromRegistry", + }); + } + } + if (seen.has(dep)) { + continue; + } + seen.add(dep); + } + if (statuses[target]) { + if (!artifacts[target].publishTarget) { + plan.push({ + artifact: target, + action: statuses[target], + }); + } + if (statuses[target] === "buildLocally" && publish) { + plan.push({ + artifact: target, + action: "publishToRegistry", + }); + } + } + seen.add(target); + } + const shortnames = { + buildLocally: "rebuild", + downloadFromRegistry: "download", + publishToRegistry: "publish", + }; + const totals = {}; + for (let { action } of plan) { + action = shortnames[action]; + totals[action] = (totals[action] || 0) + 1; + } + console.log(); + if (plan.length === 0) { + console.log("No updates needed."); + return; + } + printTable( + plan.map(({ artifact, action }) => { + const desc = { + buildLocally: " rebuild", + downloadFromRegistry: "download", + publishToRegistry: " publish", + }[action]; + if (!desc) { + throw new Error(`unexpected action key ${action}`); + } + return { artifact, desc }; + }), + [ + { key: "artifact", title: "Artifact" }, + { key: "desc", title: "Action" }, + ] + ); + console.log(); + console.log( + `Plan: ` + + ["download", "rebuild", "publish"] + .filter((action) => totals[action]) + .map((action) => `${totals[action]} to ${action}`) + .join(", ") + ); + console.log(); + console.log("Do you want to perform these actions?"); + console.log(" Depgraph will perform the actions described above."); + console.log(" Only 'yes' will be accepted to approve."); + console.log(); + const resp = await getUserInput(" Enter a value: "); + if (resp !== "yes") { + console.log(); + console.log("Apply cancelled."); + process.exit(1); + } + for (let idx = 0; idx < plan.length; idx++) { + const { artifact, action } = plan[idx]; + console.log(); + console.log( + `===== Step ${idx + 1} of ${plan.length}: ${ + shortnames[action] + } ${artifact} =====` + ); + console.log(); + await artifacts[artifact][action](info); + } + console.log(); + console.log("Apply successful!"); } async function main() { @@ -417,10 +594,11 @@ async function main() { program.option("--list", "list available artifacts; ignore other arguments"); program.option("--manual", "operate explicitly on manual artifacts"); program.option("--hold-manual", "prefer local versions of manual artifacts"); + program.option("--all", "do not skip unneeded intermediate artifacts"); program.option("--publish", "publish artifacts to remote registries"); program.option("--yes", "execute plan without confirmation"); program.parse(process.argv); - const { list, manual, holdManual, publish, yes } = program.opts(); + const { list, manual, holdManual, all, publish, yes } = program.opts(); const depgraph = await getDepGraph(); if (list) { for (const { name } of depgraph.artifacts) { @@ -437,6 +615,7 @@ async function main() { depgraph, manual, holdManual, + all, publish, yes, targets: program.args, From 0e5fd407b9f6867a1d3b1ed957fd488071ba507b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 27 Mar 2021 16:16:25 -0700 Subject: [PATCH 023/104] Misc fixes --- tools/depgraph.js | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 526fa3a..534d017 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -130,7 +130,11 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { return await hashDockerfile(name, dependentDockerHashes, { salt }); }, buildLocally: async () => { - await runCommand(`make image I=${tag}`); + if (isLangImage) { + await runCommand(`make image I=${name} L=${isLangImage.lang}`); + } else { + await runCommand(`make image I=${name}`); + } }, retrieveFromRegistry: async () => { await runCommand(`make pull I=${tag}`); @@ -563,15 +567,19 @@ async function executeDepGraph({ .join(", ") ); console.log(); - console.log("Do you want to perform these actions?"); - console.log(" Depgraph will perform the actions described above."); - console.log(" Only 'yes' will be accepted to approve."); - console.log(); - const resp = await getUserInput(" Enter a value: "); - if (resp !== "yes") { + if (yes) { + console.log("Skipping confirmation since --yes was passed."); + } else { + console.log("Do you want to perform these actions?"); + console.log(" Depgraph will perform the actions described above."); + console.log(" Only 'yes' will be accepted to approve."); console.log(); - console.log("Apply cancelled."); - process.exit(1); + const resp = await getUserInput(" Enter a value: "); + if (resp !== "yes") { + console.log(); + console.log("Apply cancelled."); + process.exit(1); + } } for (let idx = 0; idx < plan.length; idx++) { const { artifact, action } = plan[idx]; From 67299ce6feb2fe7a053fd2b21717d10908f6948e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 27 Mar 2021 16:27:50 -0700 Subject: [PATCH 024/104] Migrate S3 bucket --- Makefile | 6 +++--- tf/infra.tf | 20 ++++++++++---------- tools/depgraph.js | 4 ++-- tools/generate-build-script.js | 4 +--- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 254ef11..08372ea 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ export BUILD := build/$(T)/$(L) DEB := riju-$(T)-$(L).deb -S3_DEBS := s3://$(S3_BUCKET)-debs +S3_DEBS := s3://$(S3_BUCKET) S3_DEB := $(S3_DEBS)/debs/$(DEB) S3_HASH := $(S3_DEBS)/hashes/riju-$(T)-$(L) @@ -146,9 +146,9 @@ pkg-deb: # L= T= [Z=gzip|xz] : Build .deb from packaging environment @: $${L} $${T} fakeroot dpkg-deb --build -Z$(Z) $(BUILD)/pkg $(BUILD)/$(DEB) -## This is equivalent to the sequence 'pkg-clean', 'pkg-build', 'pkg-deb'. +## This is equivalent to the sequence 'script', 'pkg-clean', 'pkg-build', 'pkg-deb'. -pkg: pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb +pkg: script pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb ## This is equivalent to 'make pkg T=lang', 'make pkg T=config'. For ## shared dependencies, use 'make pkg T=shared' directly. diff --git a/tf/infra.tf b/tf/infra.tf index f335b7b..ee2fc92 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -43,7 +43,7 @@ data "aws_iam_policy_document" "deploy" { ] resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}", + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}", ] } @@ -53,7 +53,7 @@ data "aws_iam_policy_document" "deploy" { ] resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}/*", + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*", ] } } @@ -69,7 +69,7 @@ resource "aws_iam_user_policy_attachment" "deploy" { policy_arn = aws_iam_policy.deploy.arn } -data "aws_iam_policy_document" "riju_debs" { +data "aws_iam_policy_document" "riju" { statement { principals { type = "*" @@ -81,7 +81,7 @@ data "aws_iam_policy_document" "riju_debs" { ] resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}", + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}", ] } @@ -96,19 +96,19 @@ data "aws_iam_policy_document" "riju_debs" { ] resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju_debs.bucket}/*", + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*", ] } } -resource "aws_s3_bucket" "riju_debs" { - bucket = "${data.external.env.result.S3_BUCKET}-debs" +resource "aws_s3_bucket" "riju" { + bucket = data.external.env.result.S3_BUCKET tags = local.tags } -resource "aws_s3_bucket_policy" "riju_debs" { - bucket = aws_s3_bucket.riju_debs.id - policy = data.aws_iam_policy_document.riju_debs.json +resource "aws_s3_bucket_policy" "riju" { + bucket = aws_s3_bucket.riju.id + policy = data.aws_iam_policy_document.riju.json } data "aws_ami" "server" { diff --git a/tools/depgraph.js b/tools/depgraph.js index 534d017..7e888f7 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -37,7 +37,7 @@ function getInformationalDependencies() { JSON.parse( ( await runCommand( - `aws s3api list-objects-v2 --bucket riju-debs --prefix hashes`, + `aws s3api list-objects-v2 --bucket '${getS3Bucket()}' --prefix hashes`, { getStdout: true } ) ).stdout || '{"Contents": []}' @@ -52,7 +52,7 @@ function getInformationalDependencies() { JSON.parse( ( await runCommand( - `aws s3api list-objects-v2 --bucket riju-debs --prefix test-hashes/lang`, + `aws s3api list-objects-v2 --bucket '${getS3Bucket()}' --prefix test-hashes/lang`, { getStdout: true } ) ).stdout || '{"Contents": []}' diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index 0120c74..ed6f009 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -349,9 +349,7 @@ async function main() { "package category (lang, config, shared)" ); program.parse(process.argv); - console.log( - await generateBuildScript({ lang: program.lang, type: program.type }) - ); + console.log(await generateBuildScript(program.opts())); process.exit(0); } From 0c611c1fbaf077f061b07f2270fdb9af22e58133 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 27 Mar 2021 16:29:38 -0700 Subject: [PATCH 025/104] It lives! --- tools/depgraph.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 7e888f7..981b951 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -511,7 +511,10 @@ async function executeDepGraph({ seen.add(dep); } if (statuses[target]) { - if (!artifacts[target].publishTarget) { + if ( + !artifacts[target].publishTarget && + !(statuses[target] === "publishToRegistry" && !publish) + ) { plan.push({ artifact: target, action: statuses[target], From 4404657909a500344307eafeee528a6f1002fb2b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 22 Apr 2021 21:05:05 -0700 Subject: [PATCH 026/104] Add --local-only mode to depgraph --- tools/depgraph.js | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 981b951..c94b419 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -344,6 +344,7 @@ async function executeDepGraph({ manual, holdManual, all, + localOnly, publish, yes, targets, @@ -362,10 +363,16 @@ async function executeDepGraph({ const transitiveTargets = getTransitiveDependencies({ artifacts, targets }); const requiredInfo = new Set(); for (const target of transitiveTargets) { - for (const name of Object.values( + for (const [method, name] of Object.entries( artifacts[target].informationalDependencies || {} )) { - requiredInfo.add(name); + if ( + !( + localOnly && + ["getPublishedHash", "retrieveFromRegistry"].includes(method) + ) + ) + requiredInfo.add(name); } } const info = {}; @@ -391,7 +398,9 @@ async function executeDepGraph({ promises.desired[target] = Promise.resolve(null); } else { promises.local[target] = artifacts[target].getLocalHash(info); - promises.published[target] = artifacts[target].getPublishedHash(info); + promises.published[target] = localOnly + ? Promise.resolve(null) + : artifacts[target].getPublishedHash(info); promises.desired[target] = (async () => { const dependencyHashes = {}; for (const dependency of artifacts[target].dependencies) { @@ -424,9 +433,6 @@ async function executeDepGraph({ } await Promise.all( transitiveTargets.map(async (target) => { - const { getLocalHash, getPublishedHash, getDesiredHash } = artifacts[ - target - ]; await Promise.all([ promises.local[target].then((hash) => { hashes.local[target] = hash; @@ -600,16 +606,28 @@ async function executeDepGraph({ } async function main() { - const program = new Command(); + const program = new Command("dep"); program.usage("..."); program.option("--list", "list available artifacts; ignore other arguments"); program.option("--manual", "operate explicitly on manual artifacts"); program.option("--hold-manual", "prefer local versions of manual artifacts"); program.option("--all", "do not skip unneeded intermediate artifacts"); + program.option( + "--local-only", + "do not fetch artifacts from remote registries" + ); program.option("--publish", "publish artifacts to remote registries"); program.option("--yes", "execute plan without confirmation"); program.parse(process.argv); - const { list, manual, holdManual, all, publish, yes } = program.opts(); + const { + list, + manual, + holdManual, + all, + localOnly, + publish, + yes, + } = program.opts(); const depgraph = await getDepGraph(); if (list) { for (const { name } of depgraph.artifacts) { @@ -627,6 +645,7 @@ async function main() { manual, holdManual, all, + localOnly, publish, yes, targets: program.args, From 46b2bcf4bd50ee4d21abf07d9e9ecb3346caf249 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 22 Apr 2021 21:22:49 -0700 Subject: [PATCH 027/104] Apparently pluses are disallowed in Docker tags --- langs/{a+.yaml => aplus.yaml} | 4 +++- langs/{aspectc++.yaml => aspectcpp.yaml} | 5 +++-- langs/{c++.yaml => cpp.yaml} | 5 +++-- langs/{objectivec++.yaml => objectivecpp.yaml} | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) rename langs/{a+.yaml => aplus.yaml} (89%) rename langs/{aspectc++.yaml => aspectcpp.yaml} (84%) rename langs/{c++.yaml => cpp.yaml} (97%) rename langs/{objectivec++.yaml => objectivecpp.yaml} (93%) diff --git a/langs/a+.yaml b/langs/aplus.yaml similarity index 89% rename from langs/a+.yaml rename to langs/aplus.yaml index b6e257d..6c0552a 100644 --- a/langs/a+.yaml +++ b/langs/aplus.yaml @@ -1,5 +1,7 @@ -id: "a+" +id: "aplus" aliases: + - "a+" + - "ap" - "aplus" name: "A+" diff --git a/langs/aspectc++.yaml b/langs/aspectcpp.yaml similarity index 84% rename from langs/aspectc++.yaml rename to langs/aspectcpp.yaml index 0d5a5bd..453ba3b 100644 --- a/langs/aspectc++.yaml +++ b/langs/aspectcpp.yaml @@ -1,6 +1,7 @@ -id: "aspectc++" +id: "aspectcpp" aliases: - - "aspectcpp" + - "aspectc++" + - "aspectcplusplus" name: "AspectC++" install: diff --git a/langs/c++.yaml b/langs/cpp.yaml similarity index 97% rename from langs/c++.yaml rename to langs/cpp.yaml index 895c640..ea7382d 100644 --- a/langs/c++.yaml +++ b/langs/cpp.yaml @@ -1,6 +1,6 @@ -id: "c++" +id: "cpp" aliases: - - "cpp" + - "c++" - "g++" - "clang++" - "c++98" @@ -18,6 +18,7 @@ aliases: - "hpp" - "cxx" - "hxx" + - "cplusplus" name: "C++" monacoLang: cpp diff --git a/langs/objectivec++.yaml b/langs/objectivecpp.yaml similarity index 93% rename from langs/objectivec++.yaml rename to langs/objectivecpp.yaml index f97db77..201415d 100644 --- a/langs/objectivec++.yaml +++ b/langs/objectivecpp.yaml @@ -1,8 +1,9 @@ -id: "objectivec++" +id: "objectivecpp" aliases: - "objc++" - "objcpp" - - "objectivecpp" + - "objectivec++" + - "objectivecplusplus" name: "Objective-C++" install: From d521eed1ced8d08bd4d479ed2901b277e43651f2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 23 Apr 2021 20:33:57 -0700 Subject: [PATCH 028/104] Some random bugfixes --- backend/test-runner.js | 2 +- tools/depgraph.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/test-runner.js b/backend/test-runner.js index c5be3d6..b7b60b3 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -733,7 +733,7 @@ async function main() { passed.forEach((_, { lang }) => { langsValidated[lang] = true; }); - failed.forEach(({ lang }) => { + failed.forEach((_, { lang }) => { langsValidated[lang] = false; }); for (const [lang, validated] of Object.entries(langsValidated)) { diff --git a/tools/depgraph.js b/tools/depgraph.js index c94b419..75afee1 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -640,6 +640,7 @@ async function main() { if (program.args.length === 0) { program.help({ error: true }); } + await runCommand("make all-scripts"); await executeDepGraph({ depgraph, manual, From db41dfa0a34da3e004d635721d76fdb5feb4f15d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 24 Apr 2021 09:21:10 -0700 Subject: [PATCH 029/104] Container process should have a pty --- backend/api.js | 52 ++++++++++++++--------------- system/src/riju-system-privileged.c | 2 +- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/backend/api.js b/backend/api.js index 9606592..551f99e 100644 --- a/backend/api.js +++ b/backend/api.js @@ -56,33 +56,33 @@ export class Session { try { allSessions.add(this); const containerArgs = this.privilegedSession(); - const containerProc = spawn(containerArgs[0], containerArgs.slice(1)); + const containerPty = pty.spawn(containerArgs[0], containerArgs.slice(1), { + name: "xterm-color", + }); this.container = { - proc: containerProc, + pty: containerPty, }; - for (const stream of [containerProc.stdout, containerProc.stderr]) { - stream.on("data", (data) => - this.send({ - event: "serviceLog", - service: "container", - output: data.toString("utf8"), - }) - ); - containerProc.on("close", (code, signal) => - this.send({ - event: "serviceFailed", - service: "container", - error: `Exited with status ${signal || code}`, - }) - ); - containerProc.on("error", (err) => - this.send({ - event: "serviceFailed", - service: "container", - error: `${err}`, - }) - ); - } + containerPty.on("data", (data) => + this.send({ + event: "serviceLog", + service: "container", + output: data.toString("utf8"), + }) + ); + containerPty.on("close", (code, signal) => + this.send({ + event: "serviceFailed", + service: "container", + error: `Exited with status ${signal || code}`, + }) + ); + containerPty.on("error", (err) => + this.send({ + event: "serviceFailed", + service: "container", + error: `${err}`, + }) + ); await this.run(this.privilegedWait(this.context)); if (this.config.setup) { await this.run(this.privilegedExec(bash(this.config.setup))); @@ -438,7 +438,7 @@ export class Session { this.log(`Tearing down session`); this.tearingDown = true; if (this.container) { - this.container.proc.stdin.end(); + this.container.pty.kill(); } allSessions.delete(this); this.ws.terminate(); diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index afd7210..7614c84 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -53,7 +53,7 @@ void session(char *uuid, char *lang) char *argv[] = { "docker", "run", - "--rm", "-i", + "--rm", "-it", "-e", "HOME=/home/riju", "-e", "HOSTNAME=riju", "-e", "LANG=C.UTF-8", From 0e0ceebd9e98ceeb5409d6e026a004e4e0f7e329 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 24 Apr 2021 09:38:51 -0700 Subject: [PATCH 030/104] Workaround Docker bug with term size settings --- backend/util.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/util.js b/backend/util.js index aebbd1b..5d36faf 100644 --- a/backend/util.js +++ b/backend/util.js @@ -66,6 +66,8 @@ export function bash(cmdline) { // single command (no shell logic). cmdline = "exec " + cmdline; } + // Workaround https://github.com/moby/moby/issues/25450 + cmdline = "stty cols 80 rows 24; " + cmdline; return ["bash", "-c", `set -euo pipefail; ${cmdline}`]; } From d3cd61cdd6454508932db47905ed3fc312e931ef Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 24 Apr 2021 09:48:25 -0700 Subject: [PATCH 031/104] Make stty conditional --- backend/api.js | 43 +++++++++++++++++-------------------------- backend/util.js | 11 ++++++++--- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/backend/api.js b/backend/api.js index 551f99e..a39fa23 100644 --- a/backend/api.js +++ b/backend/api.js @@ -49,8 +49,10 @@ export class Session { privilegedSession = () => util.privilegedSession(this.context); privilegedWait = () => util.privilegedWait(this.context); - privilegedExec = (args) => util.privilegedExec(this.context, args); - privilegedPty = (args) => util.privilegedPty(this.context, args); + privilegedExec = (cmdline) => + util.privilegedExec(this.context, bash(cmdline)); + privilegedPty = (cmdline) => + util.privilegedPty(this.context, bash(cmdline, { stty: true })); setup = async () => { try { @@ -85,11 +87,11 @@ export class Session { ); await this.run(this.privilegedWait(this.context)); if (this.config.setup) { - await this.run(this.privilegedExec(bash(this.config.setup))); + await this.run(this.privilegedExec(this.config.setup)); } await this.runCode(); if (this.config.daemon) { - const daemonArgs = this.privilegedExec(bash(this.config.daemon)); + const daemonArgs = this.privilegedExec(this.config.daemon); const daemonProc = spawn(daemonArgs[0], daemonArgs.slice(1)); this.daemon = { proc: daemonProc, @@ -120,9 +122,9 @@ export class Session { } if (this.config.lsp) { if (this.config.lsp.setup) { - await this.run(this.privilegedExec(bash(this.config.lsp.setup))); + await this.run(this.privilegedExec(this.config.lsp.setup)); } - const lspArgs = this.privilegedExec(bash(this.config.lsp.start)); + const lspArgs = this.privilegedExec(this.config.lsp.start); const lspProc = spawn(lspArgs[0], lspArgs.slice(1)); this.lsp = { proc: lspProc, @@ -266,22 +268,11 @@ export class Session { writeCode = async (code) => { if (this.config.main.includes("/")) { - await this.run( - this.privilegedExec([ - "mkdir", - "-p", - path.dirname(`${this.homedir}/${this.config.main}`), - ]) - ); + const dir = path.dirname(`${this.homedir}/${this.config.main}`); + await this.run(this.privilegedExec(`mkdir -p ${dir}`)); } - await this.run( - this.privilegedExec([ - "sh", - "-c", - `cat > ${path.resolve(this.homedir, this.config.main)}`, - ]), - { input: code } - ); + const file = path.resolve(this.homedir, this.config.main); + await this.run(this.privilegedExec(`cat > ${file}`), { input: code }); }; runCode = async (code) => { @@ -299,7 +290,7 @@ export class Session { if (this.term) { const pid = this.term.pty.pid; const args = this.privilegedExec( - bash(`kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}`) + `kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}` ); spawn(args[0], args.slice(1)); // Signal to terminalOutput message generator using closure. @@ -325,7 +316,7 @@ export class Session { code += suffix + "\n"; } await this.writeCode(code); - const termArgs = this.privilegedPty(bash(cmdline)); + const termArgs = this.privilegedPty(cmdline); const term = { pty: pty.spawn(termArgs[0], termArgs.slice(1), { name: "xterm-color", @@ -365,13 +356,13 @@ export class Session { if (this.formatter) { const pid = this.formatter.proc.pid; const args = this.privilegedExec( - bash(`kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}`) + `kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}` ); spawn(args[0], args.slice(1)); this.formatter.live = false; this.formatter = null; } - const args = this.privilegedExec(bash(this.config.format.run)); + const args = this.privilegedExec(this.config.format.run); const formatter = { proc: spawn(args[0], args.slice(1)), live: true, @@ -424,7 +415,7 @@ export class Session { }; ensure = async (cmd) => { - const code = await this.run(this.privilegedExec(bash(cmd)), { + const code = await this.run(this.privilegedExec(cmd), { check: false, }); this.send({ event: "ensured", code }); diff --git a/backend/util.js b/backend/util.js index 5d36faf..c2f5504 100644 --- a/backend/util.js +++ b/backend/util.js @@ -60,14 +60,19 @@ export function privilegedPty({ uuid }, args) { return [rijuSystemPrivileged, "pty", uuid].concat(args); } -export function bash(cmdline) { +export function bash(cmdline, opts) { + const stty = opts && opts.stty; if (!cmdline.match(/[;|&(){}=\n]/)) { // Reduce number of subshells we generate, if we're just running a // single command (no shell logic). cmdline = "exec " + cmdline; } - // Workaround https://github.com/moby/moby/issues/25450 - cmdline = "stty cols 80 rows 24; " + cmdline; + if (stty) { + // Workaround https://github.com/moby/moby/issues/25450 (the issue + // thread claims the bug is resolved and released, but not in my + // testing). + cmdline = "stty cols 80 rows 24; " + cmdline; + } return ["bash", "-c", `set -euo pipefail; ${cmdline}`]; } From 6c7fbf7fb12cc331d04199a2481127883132e3f3 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 30 Apr 2021 23:13:15 -0700 Subject: [PATCH 032/104] Start fixing sandbox script --- Makefile | 2 +- backend/langs.js | 7 +++--- backend/sandbox.bash | 7 ++++-- backend/sandbox.js | 55 ++++++++++++++++++++++++++++++-------------- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 08372ea..0ee66a2 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ else ifneq (,$(filter $(I),base lang)) ifeq ($(I),lang) @: $${L} endif - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) + docker run -it --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) else ifeq ($(I),runtime) docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) else diff --git a/backend/langs.js b/backend/langs.js index 63716b0..f477050 100644 --- a/backend/langs.js +++ b/backend/langs.js @@ -13,10 +13,9 @@ export let langs = {}; // Map from language aliases and IDs to canonical language IDs. export let aliases = {}; -// Read languages from JSON files in /opt/riju/langs, and update the -// global langs variable in this module. Never throw an error. If -// there is a problem then just leave the languages as they previously -// were. +// Read languages from YAML, and update the global langs variable in +// this module. Never throw an error. If there is a problem then just +// leave the languages as they previously were. async function updateLangsFromDisk() { try { const newLangs = {}; diff --git a/backend/sandbox.bash b/backend/sandbox.bash index a08e84d..8c6f6ec 100644 --- a/backend/sandbox.bash +++ b/backend/sandbox.bash @@ -5,10 +5,13 @@ if [[ -z "$L" ]]; then exit 1 fi -cfg="$(< "/opt/riju/langs/$L.json")" || exit 1 +if [[ -z "$LANG_CONFIG" ]]; then + echo 'environment variable unset: $LANG_CONFIG' >&2 + exit 1 +fi function get { - jq -r ".$1" <<< "${cfg}" + jq -r ".$1" <<< "${LANG_CONFIG}" } function has { diff --git a/backend/sandbox.js b/backend/sandbox.js index a89d79e..19fb2fd 100644 --- a/backend/sandbox.js +++ b/backend/sandbox.js @@ -2,14 +2,17 @@ import { spawn } from "child_process"; import { promises as fs } from "fs"; import process from "process"; +import pty from "node-pty"; import { quote } from "shell-quote"; -import { getUUID } from "./util.js"; - +import { readLangConfig } from "../lib/yaml.js"; import { - privilegedSetup, - privilegedSpawn, - privilegedTeardown, + bash, + getUUID, + privilegedExec, + privilegedPty, + privilegedSession, + privilegedWait, run, } from "./util.js"; @@ -28,22 +31,40 @@ async function main() { if (!lang) { die("environment variable unset: $L"); } + const langConfig = await readLangConfig(lang); const uuid = getUUID(); - await run(privilegedSetup({ uuid }), log); - const args = privilegedSpawn({ uuid }, [ - "bash", - "-c", - `exec env L='${lang}' bash --rcfile <(cat <<< ${quote([sandboxScript])})`, - ]); + console.log(`Starting session with UUID ${uuid}`); + const sessionArgs = privilegedSession({ uuid, lang }); + const session = pty.spawn(sessionArgs[0], sessionArgs.slice(1), { + name: "xterm-color", + }); + await run(privilegedWait({ uuid }), log); + console.log( + bash( + `env L='${lang}' LANG_CONFIG=${quote([ + JSON.stringify(langConfig), + ])} bash --rcfile <(cat <<< ${quote([sandboxScript])})` + )[2] + ); + const args = privilegedPty( + { uuid }, + bash( + `env L='${lang}' LANG_CONFIG=${quote([ + JSON.stringify(langConfig), + ])} bash --rcfile <(cat <<< ${quote([sandboxScript])})` + ) + ); const proc = spawn(args[0], args.slice(1), { stdio: "inherit", }); - await new Promise((resolve, reject) => { - proc.on("error", reject); - proc.on("close", resolve); - }); - await run(privilegedTeardown({ uuid }), log); - await returnUser(); + try { + await new Promise((resolve, reject) => { + proc.on("error", reject); + proc.on("close", resolve); + }); + } finally { + session.kill(); + } } main().catch(die); From 49b00dcab241f0f954fc6fda5f683f8ecb2f97eb Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 21:23:06 -0700 Subject: [PATCH 033/104] Mark output as sensitive --- tf/infra.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tf/infra.tf b/tf/infra.tf index ee2fc92..4686276 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -192,5 +192,6 @@ output "deploy_aws_access_key_id" { } output "deploy_aws_secret_access_key" { - value = aws_iam_access_key.deploy.secret + value = aws_iam_access_key.deploy.secret + sensitive = true } From b9e0d4049c14cbdf89eef1adb5e9c56267a46dfa Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 21:23:09 -0700 Subject: [PATCH 034/104] Add ECR repository back into Terraform --- tf/infra.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tf/infra.tf b/tf/infra.tf index 4686276..747da42 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -111,6 +111,11 @@ resource "aws_s3_bucket_policy" "riju" { policy = data.aws_iam_policy_document.riju.json } +resource "aws_ecr_repository" "riju" { + name = "riju" + image_tag_mutability = "IMMUTABLE" +} + data "aws_ami" "server" { owners = ["self"] From 2ba198b6451483dcbe54b25a9d0bdaa2c1592840 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 21:25:36 -0700 Subject: [PATCH 035/104] "Fix" race condition in riju-init-volume --- packer/riju-init-volume | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packer/riju-init-volume b/packer/riju-init-volume index 520b748..945695e 100755 --- a/packer/riju-init-volume +++ b/packer/riju-init-volume @@ -27,6 +27,8 @@ print "volume has ${num_parts} partition(s)" if [[ "${num_parts}" != 1 ]]; then print "repartitioning so we have exactly one partition" sfdisk -X gpt "/dev/${disk}" <<< ";" + print "waiting for 1 second so that partitions show up in /dev" + sleep 1 fi part="$(lsblk -l -o name | (grep "${disk}." || true) | head -n1)" From bcd8551e55015f22cf4ba75f935f2ff94ed73bb3 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 21:25:46 -0700 Subject: [PATCH 036/104] Foolproof --- tools/depgraph.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/depgraph.js b/tools/depgraph.js index 75afee1..e9cca15 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -349,6 +349,7 @@ async function executeDepGraph({ yes, targets, }) { + await runCommand(`make all-scripts`); const artifacts = {}; for (const artifact of depgraph.artifacts) { for (const dep of artifact.dependencies) { From 6c678c73ce04849bfa588e366955674515ea3541 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 22:17:08 -0700 Subject: [PATCH 037/104] Start seriously screwing around with GBS.js --- Makefile | 36 +------ docker/base/install.bash | 71 +------------ docker/lang/install.bash | 9 ++ docker/packaging/install.bash | 20 +--- langs/ceylon.yaml | 8 ++ langs/crystal.yaml | 4 + langs/d.yaml | 7 +- langs/dart.yaml | 4 + langs/hack.yaml | 4 + langs/mongodb.yaml | 9 +- langs/qsharp.yaml | 9 +- langs/r.yaml | 4 + lib/yaml.js | 33 +++--- tools/depgraph.js | 1 - tools/generate-build-script.js | 184 ++++++++++++++++++++++++--------- tools/jsonschema.yaml | 29 ++++++ tools/make-foreach.js | 5 - 17 files changed, 235 insertions(+), 202 deletions(-) diff --git a/Makefile b/Makefile index 0ee66a2..0db51f8 100644 --- a/Makefile +++ b/Makefile @@ -96,28 +96,17 @@ repkg: script # L= T= : Build fresh .deb and install into live conta $(MAKE_QUIETLY) shell I=packaging CMD="make pkg L=$(L) T=$(T)" ctr="$$(docker container ls -f label="riju-install-target=yes" -l -q)"; test "$${ctr}" || (echo "no valid container is live"; exit 1); docker exec "$${ctr}" make install L=$(L) T=$(T) -## This is equivalent to 'make repkg T=lang', 'make repkg T=config'. -## For shared dependencies, use 'make repkg T=shared' directly. - -repkgs: # L= : Build and install fresh lang and config .debs - @: $${L} - node tools/make-foreach.js --types repkg L=$(L) - ### Build packaging scripts script: # L= T= : Generate a packaging script @: $${L} $${T} mkdir -p $(BUILD) node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash +ifeq ($(T),lang) + node tools/generate-build-script.js --lang $(L) --type install > $(BUILD)/install.bash +endif chmod +x $(BUILD)/build.bash -scripts: # L= : Generate both lang and config packaging scripts - @: $${L} - node tools/make-foreach.js --types script L=$(L) - -## This is equivalent to 'make script T=lang', 'make script T=config'. -## For shared dependencies, use 'make script T=shared' directly. - all-scripts: # Generate packaging scripts for all languages node tools/write-all-build-scripts.js @@ -146,20 +135,9 @@ pkg-deb: # L= T= [Z=gzip|xz] : Build .deb from packaging environment @: $${L} $${T} fakeroot dpkg-deb --build -Z$(Z) $(BUILD)/pkg $(BUILD)/$(DEB) -## This is equivalent to the sequence 'script', 'pkg-clean', 'pkg-build', 'pkg-deb'. +## This is equivalent to the sequence 'pkg-clean', 'pkg-build', 'pkg-deb'. -pkg: script pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb - -## This is equivalent to 'make pkg T=lang', 'make pkg T=config'. For -## shared dependencies, use 'make pkg T=shared' directly. -# -## Z is the compression type to use; defaults to none. Higher -## compression levels (gzip is moderate, xz is high) take much longer -## but produce much smaller packages. - -pkgs: # L= [Z=gzip|xz] : Build both lang and config .debs - @: $${L} - node tools/make-foreach.js --types pkg L=$(L) +pkg: pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb ### Install packages @@ -168,10 +146,6 @@ install: # L= T= : Install built .deb if [[ -z "$$(ls -A /var/lib/apt/lists)" ]]; then sudo apt update; fi DEBIAN_FRONTEND=noninteractive sudo -E apt reinstall -y ./$(BUILD)/$(DEB) -installs: # L= : Install both lang and config .debs - @: $${L} - node tools/make-foreach.js --types install L=$(L) - ### Build and run application code frontend: # Compile frontend assets for production diff --git a/docker/base/install.bash b/docker/base/install.bash index ddf82b3..5bbc922 100755 --- a/docker/base/install.bash +++ b/docker/base/install.bash @@ -11,99 +11,30 @@ pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive -dpkg --add-architecture i386 - apt-get update +apt-get dist-upgrade -y (yes || true) | unminimize apt-get install -y curl gnupg lsb-release wget -# Ceylon -wget https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem -O /usr/local/share/ca-certificates/DigiCertTLSRSASHA2562020CA1.crt - -# D -wget https://letsencrypt.org/certs/lets-encrypt-r3.pem -O /usr/local/share/ca-certificates/lets-encrypt-r3.crt - -update-ca-certificates - -ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" -cran_repo="$(curl -fsSL https://cran.r-project.org/bin/linux/ubuntu/ | grep -Eo 'cran[0-9]+' | head -n1)" node_repo="$(curl -fsSL https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" -# .NET -wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" -apt-get install ./packages-microsoft-prod.deb - -# Ceylon -curl -fsSL https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key | apt-key add - - -# Crystal -curl -fsSL https://keybase.io/crystal/pgp_keys.asc | apt-key add - - -# Dart -curl -fsSL https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - - -# Hack -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B4112585D386EB94 - -# MongoDB -curl -fsSL https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - - # Node.js curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - -# R -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 - # Yarn curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null < "install-lang-${LANG}.bash" riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" ( @@ -21,6 +22,7 @@ riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" (grep -Eo 'riju-shared-[^, ]+' || true) | sed 's/riju-shared-//' ) | while read name; do + riju-curl "build/shared/${name}/install.bash" > "install-shared-${name}.bash" riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb" done @@ -28,12 +30,19 @@ if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then apt-get update fi +if compgen -G "./install-shared-*.bash"; then + for file in ./install-shared-*.bash; do + "${file}" + done +fi + if compgen -G "./riju-shared-*.deb"; then for file in ./riju-shared-*.deb; do apt-get install -y "${file}" done fi +"./install-lang-${LANG}.bash" apt-get install -y "./riju-lang-${LANG}.deb" popd diff --git a/docker/packaging/install.bash b/docker/packaging/install.bash index 1b08ef3..b3a0aa7 100755 --- a/docker/packaging/install.bash +++ b/docker/packaging/install.bash @@ -10,42 +10,24 @@ pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive -dpkg --add-architecture i386 - apt-get update +apt-get dist-upgrade -y (yes || true) | unminimize apt-get install -y curl gnupg lsb-release wget -wget https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem -O /usr/local/share/ca-certificates/DigiCertTLSRSASHA2562020CA1.crt -wget https://letsencrypt.org/certs/lets-encrypt-r3.pem -O /usr/local/share/ca-certificates/lets-encrypt-r3.crt - -update-ca-certificates - -ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" -wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" -apt-get install ./packages-microsoft-prod.deb - -curl -fsSL https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key | apt-key add - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null </dev/null < pkg.includes("$")).length > 0)) ) { parts.push(`\ -export DEBIAN_FRONTEND=noninteractive -sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); +export DEBIAN_FRONTEND=noninteractive`); + if ( + install.prepare && + ((install.prepare.manual && + install.prepare.manual.includes("apt-get") && + install.prepare.manual.includes(":i386")) || + (install.prepare.apt && + install.prepare.apt.filter((pkg) => pkg.includes(":i386")).length > + 0)) + ) { + parts.push(`\ +dpkg --add-architecture i386`); + } + parts.push(`\ + sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); } if (install) { const { @@ -49,23 +64,67 @@ sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); deb, } = install; if (prepare) { - const { apt, npm, opam, manual } = prepare; + const { + preface, + cert, + aptKey, + aptRepo, + apt, + npm, + opam, + manual, + } = prepare; + if (preface) { + prefaceParts.push(preface); + } + if (cert && cert.length > 0) { + prefaceParts.push( + cert + .map( + (url, idx) => + `sudo wget "${url}" -O /usr/local/share/ca-certificates/riju-${id}-${idx}.crt` + ) + .join("\n") + ); + prefaceParts.push(`sudo update-ca-certificates`); + } + if (aptKey && aptKey.length > 0) { + prefaceParts.push( + aptKey + .map((src) => { + if (src.startsWith("http://") || src.startsWith("https://")) { + return `curl -fsSL "${src}" | sudo apt-key add -`; + } else if (/^[0-9A-F]+$/.match(src)) { + return `sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${src}"`; + } else { + throw new Error(`unknown aptKey format: ${src}`); + } + }) + .join("\n") + ); + } + if (aptRepo && aptRepo.length > 0) { + prefaceParts.push(`sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null < 0) { - parts.push(`\ + needsAptGetUpdate = true; + prefaceParts.push(`\ sudo --preserve-env=DEBIAN_FRONTEND apt-get install -y ${apt.join(" ")}`); } if (npm && npm.length > 0) { - parts.push(`\ + prefaceParts.push(`\ sudo npm install -g ${npm.join(" ")}`); } if (opam && opam.length > 0) { - parts.push(`\ + prefaceParts.push(`\ sudo opam init -n --disable-sandboxing --root /opt/opam sudo opam install "${opam.join(" ")}" -y --root /opt/opam sudo ln -s /opt/opam/default/bin/* /usr/local/bin/`); } if (manual) { - parts.push(manual); + prefaceParts.push(manual); } } if (npm && npm.length > 0) { @@ -215,6 +274,9 @@ chmod +x "${path}"`); } } if (manual) { + if (manual.includes("apt-get")) { + needsAptGetUpdate = true; + } parts.push(manual); } if (deb) { @@ -223,6 +285,9 @@ chmod +x "${path}"`); ); } if (apt) { + if (apt.filter((pkg) => pkg.includes("$")).length > 0) { + needsAptGetUpdate = true; + } depends = depends.concat(apt); } if (dependsCfg.unpin) { @@ -237,6 +302,13 @@ chmod +x "${path}"`); ); } } + if (needsAptGetUpdate) { + prefaceParts.unshift(`\ +export DEBIAN_FRONTEND=noninteractive`); + prefaceParts.push(`\ +sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); + } + parts = prefaceParts.concat(parts); parts.push(`depends=(${depends.map((dep) => `"${dep}"`).join(" ")})`); let stripDependsFilter = ""; const stripDepends = (dependsCfg.strip || []).concat(dependsCfg.unpin || []); @@ -280,40 +352,6 @@ set -euxo pipefail`); return parts.join("\n\n"); } -// Given a language config object, return the text of a Bash script -// that will build the (unpacked) riju-config-foo Debian package into -// ${pkg} when run in an appropriate environment. This is a package -// that will install configuration files and/or small scripts that -// encode the language configuration so that Riju can operate on any -// installed languages without knowing their configuration in advance. -function makeConfigScript(langConfig) { - const { id, name } = langConfig; - let parts = []; - parts.push(`\ -#!/usr/bin/env bash - -set -euxo pipefail`); - let debianControlData = `\ -Package: riju-config-${id} -Version: \$(date +%s%3N) -Architecture: all -Maintainer: Radon Rosborough -Description: Riju configuration for the ${name} language -Depends: riju-lang-${id} -Riju-Script-Hash: \$(sha1sum "$0" | awk '{ print $1 }')`; - parts.push(`\ -install -d "\${pkg}/DEBIAN" -cat < "\${pkg}/DEBIAN/control" -${debianControlData} -EOF`); - parts.push(`\ -install -d "\${pkg}/opt/riju/langs" -cat <<"EOF" > "\${pkg}/opt/riju/langs/${id}.json" -${JSON.stringify(langConfig, null, 2)} -EOF`); - return parts.join("\n\n"); -} - // Given a language config object, return the text of a Bash script // that will build the (unpacked) riju-shared-foo Debian package into // ${pkg} when run in an appropriate environment. This is a package @@ -323,20 +361,66 @@ function makeSharedScript(langConfig) { return makeLangScript(langConfig, true); } +// Given a language ID, return the text of a Bash script that will do +// any necessary setup before the language package is installed (along +// with its shared dependencies, if any). +function makeInstallScript(lang) { + let parts = []; + if (install) { + const { apt, cert, aptKey, aptRepo } = install; + if (apt && apt.filter((pkg) => pkg.includes(":i386")).length > 0) { + parts.push(`\ +dpkg --add-architecture i386`); + } + if (cert && cert.length > 0) { + parts.push( + cert + .map( + (url, idx) => + `sudo wget "${url}" -O /usr/local/share/ca-certificates/riju-${id}-${idx}.crt` + ) + .join("\n") + ); + parts.push(`sudo update-ca-certificates`); + } + if (aptKey && aptKey.length > 0) { + parts.push( + aptKey + .map((src) => { + if (src.startsWith("http://") || src.startsWith("https://")) { + return `curl -fsSL "${src}" | sudo apt-key add -`; + } else if (/^[0-9A-F]+$/.match(src)) { + return `sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${src}"`; + } else { + throw new Error(`unknown aptKey format: ${src}`); + } + }) + .join("\n") + ); + } + if (aptRepo && aptRepo.length > 0) { + parts.push(`sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null < makeLangScript(await readLangConfig(lang)), + shared: async () => makeSharedScript(await readSharedDepConfig(lang)), + install: async () => await makeInstallScript(lang), }[type]; if (!scriptMaker) { throw new Error(`unsupported script type ${type}`); } - return scriptMaker( - type === "shared" - ? await readSharedDepConfig(lang) - : await readLangConfig(lang) - ); + return scriptMaker(); } // Parse command-line arguments, run main functionality, and exit. @@ -346,7 +430,7 @@ async function main() { .requiredOption("--lang ", "language ID") .requiredOption( "--type ", - "package category (lang, config, shared)" + "package category (lang, shared, install)" ); program.parse(process.argv); console.log(await generateBuildScript(program.opts())); diff --git a/tools/jsonschema.yaml b/tools/jsonschema.yaml index dac9c91..edc3255 100644 --- a/tools/jsonschema.yaml +++ b/tools/jsonschema.yaml @@ -511,6 +511,31 @@ properties: type: object additionalProperties: false properties: + preface: &preface + type: string + minLength: 1 + cert: &cert + type: array + items: + type: string + pattern: "^https?://" + examples: + - "https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem" + aptKey: &aptKey + type: array + items: + type: string + pattern: "^https?://|^[0-9A-F]+$" + examples: + - "https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key" + - "B4112585D386EB94" + aptRepo: &aptRepo + type: array + items: + type: string + pattern: "^https?://" + examples: + - "https://downloads.ceylon-lang.org/apt/ unstable main" apt: type: array items: @@ -529,6 +554,10 @@ properties: manual: type: string minLength: 1 + preface: *preface + cert: *cert + aptKey: *aptKey + aptRepo: *aptRepo apt: type: array items: diff --git a/tools/make-foreach.js b/tools/make-foreach.js index 56fc1e5..143cadd 100644 --- a/tools/make-foreach.js +++ b/tools/make-foreach.js @@ -20,11 +20,6 @@ async function main() { ); } break; - case "--types": - for (const type of ["lang", "config"]) { - await runCommand(`MAKELEVEL= make ${targets.join(" ")} T=${type}`); - } - break; default: console.error(`make-foreach.js: unknown selector: ${selector}`); process.exit(1); From 3f898fdbc19df45fcfef9bd554d134343aa36202 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 22:30:00 -0700 Subject: [PATCH 038/104] Get Python built with new system --- docker/lang/install.bash | 13 ++++++++----- docker/runtime/install.bash | 6 +++--- tools/generate-build-script.js | 3 ++- tools/write-all-build-scripts.js | 16 ++++++++++++---- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docker/lang/install.bash b/docker/lang/install.bash index d8e7c04..b42610b 100755 --- a/docker/lang/install.bash +++ b/docker/lang/install.bash @@ -16,6 +16,7 @@ export DEBIAN_FRONTEND=noninteractive riju-curl "build/lang/${LANG}/install.bash" > "install-lang-${LANG}.bash" riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" +chmod +x "install-lang-${LANG}.bash" ( dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | @@ -24,25 +25,27 @@ riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" ) | while read name; do riju-curl "build/shared/${name}/install.bash" > "install-shared-${name}.bash" riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb" + chmod +x "install-shared-${name}.bash" done -if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then - apt-get update -fi - if compgen -G "./install-shared-*.bash"; then for file in ./install-shared-*.bash; do "${file}" done fi +"./install-lang-${LANG}.bash" + +if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then + apt-get update +fi + if compgen -G "./riju-shared-*.deb"; then for file in ./riju-shared-*.deb; do apt-get install -y "${file}" done fi -"./install-lang-${LANG}.bash" apt-get install -y "./riju-lang-${LANG}.deb" popd diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 87da6ad..73283df 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -12,6 +12,7 @@ pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive apt-get update +apt-get dist-upgrade -y (yes || true) | unminimize apt-get install -y curl gnupg lsb-release wget @@ -65,10 +66,9 @@ vim apt-get update apt-get install -y $(sed 's/#.*//' <<< "${packages}") -ver="$(latest_release watchexec/watchexec)" -wget "https://github.com/watchexec/watchexec/releases/download/${ver}/watchexec-${ver}-x86_64-unknown-linux-gnu.deb" +ver="$(latest_release watchexec/watchexec | sed 's/^cli-v//')" +wget "https://github.com/watchexec/watchexec/releases/download/cli-v${ver}/watchexec-${ver}-x86_64-unknown-linux-gnu.deb" apt-get install -y ./watchexec-*.deb -rm watchexec-*.deb rm -rf /var/lib/apt/lists/* diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index a549e9d..5779e67 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -364,8 +364,9 @@ function makeSharedScript(langConfig) { // Given a language ID, return the text of a Bash script that will do // any necessary setup before the language package is installed (along // with its shared dependencies, if any). -function makeInstallScript(lang) { +function makeInstallScript(langConfig) { let parts = []; + const { install } = langConfig; if (install) { const { apt, cert, aptKey, aptRepo } = install; if (apt && apt.filter((pkg) => pkg.includes(":i386")).length > 0) { diff --git a/tools/write-all-build-scripts.js b/tools/write-all-build-scripts.js index 06868b5..f03b394 100644 --- a/tools/write-all-build-scripts.js +++ b/tools/write-all-build-scripts.js @@ -13,13 +13,21 @@ import { generateBuildScript } from "./generate-build-script.js"; // Parse command-line arguments, run main functionality, and exit. async function main() { for (const { lang, type } of await getPackages()) { - const scriptPath = `build/${type}/${lang}/build.bash`; - await fs.mkdir(nodePath.dirname(scriptPath), { recursive: true }); + const buildScriptPath = `build/${type}/${lang}/build.bash`; + const installScriptPath = `build/${type}/${lang}/install.bash`; + await fs.mkdir(nodePath.dirname(buildScriptPath), { recursive: true }); await fs.writeFile( - scriptPath, + buildScriptPath, (await generateBuildScript({ lang, type })) + "\n" ); - await fs.chmod(scriptPath, 0o755); + await fs.chmod(buildScriptPath, 0o755); + if (type === "lang") { + await fs.writeFile( + installScriptPath, + (await generateBuildScript({ lang, type: "install" })) + "\n" + ); + await fs.chmod(installScriptPath, 0o755); + } } process.exit(0); } From 0600d3d6874141a5e2362e7687fd7189c0bcdfca Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 13:07:27 -0700 Subject: [PATCH 039/104] Fix some tags --- tf/.terraform.lock.hcl | 54 +++++++++++++++++++++--------------------- tf/infra.tf | 10 ++++++-- 2 files changed, 35 insertions(+), 29 deletions(-) mode change 100755 => 100644 tf/.terraform.lock.hcl diff --git a/tf/.terraform.lock.hcl b/tf/.terraform.lock.hcl old mode 100755 new mode 100644 index 145903a..14a64fd --- a/tf/.terraform.lock.hcl +++ b/tf/.terraform.lock.hcl @@ -2,38 +2,38 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "2.70.0" - constraints = "~> 2.70" + version = "3.45.0" + constraints = "~> 3.45" hashes = [ - "h1:6tf4jg37RrMHyVCql+fEgAFvX8JiqDognr+lk6rx7To=", - "zh:01a5f351146434b418f9ff8d8cc956ddc801110f1cc8b139e01be2ff8c544605", - "zh:1ec08abbaf09e3e0547511d48f77a1e2c89face2d55886b23f643011c76cb247", - "zh:606d134fef7c1357c9d155aadbee6826bc22bc0115b6291d483bc1444291c3e1", - "zh:67e31a71a5ecbbc96a1a6708c9cc300bbfe921c322320cdbb95b9002026387e1", - "zh:75aa59ae6f0834ed7142c81569182a658e4c22724a34db5d10f7545857d8db0c", - "zh:76880f29fca7a0a3ff1caef31d245af2fb12a40709d67262e099bc22d039a51d", - "zh:aaeaf97ffc1f76714e68bc0242c7407484c783d604584c04ad0b267b6812b6dc", - "zh:ae1f88d19cc85b2e9b6ef71994134d55ef7830fd02f1f3c58c0b3f2b90e8b337", - "zh:b155bdda487461e7b3d6e3a8d5ce5c887a047e4d983512e81e2c8266009f2a1f", - "zh:ba394a7c391a26c4a91da63ad680e83bde0bc1ecc0a0856e26e9d62a4e77c408", - "zh:e243c9d91feb0979638f28eb26f89ebadc179c57a2bd299b5729fb52bd1902f2", - "zh:f6c05e20d9a3fba76ca5f47206dde35e5b43b6821c6cbf57186164ce27ba9f15", + "h1:LKU/xfna87/p+hl5yTTW3dvOqWJp5JEM+Dt3nnvSDvA=", + "zh:0fdbb3af75ff55807466533f97eb314556ec41a908a543d7cafb06546930f7c6", + "zh:20656895744fa0f4607096b9681c77b2385f450b1577f9151d3070818378a724", + "zh:390f316d00f25a5e45ef5410961fd05bf673068c1b701dc752d11df6d8e741d7", + "zh:3da70f9de241d5f66ea9994ef1e0beddfdb005fa2d2ef6712392f57c5d2e4844", + "zh:65de63cc0f97c85c28a19db560c546aa25f4f403dbf4783ac53c3918044cf180", + "zh:6fc52072e5a66a5d0510aaa2b373a2697895f51398613c68619d8c0c95fc75f5", + "zh:7c1da61092bd1206a020e3ee340ab11be8a4f9bb74e925ca1229ea5267fb3a62", + "zh:94e533d86ce3c08e7102dcabe34ba32ae7fd7819fd0aedef28f48d29e635eae2", + "zh:a3180d4826662e19e71cf20e925a2be8613a51f2f3f7b6d2643ac1418b976d58", + "zh:c783df364928c77fd4dec5419533b125bebe2d50212c4ad609f83b701c2d981a", + "zh:e1279bde388cb675d324584d965c6d22c3ec6890b13de76a50910a3bcd84ed64", ] } provider "registry.terraform.io/hashicorp/external" { - version = "2.0.0" + version = "2.1.0" hashes = [ - "h1:Q5xqryWI3tCY8yr+fugq7dz4Qz+8g4GaW9ZS8dc6Ob8=", - "zh:07949780dd6a1d43e7b46950f6e6976581d9724102cb5388d3411a1b6f476bde", - "zh:0a4f4636ff93f0644affa8474465dd8c9252946437ad025b28fc9f6603534a24", - "zh:0dd7e05a974c649950d1a21d7015d3753324ae52ebdd1744b144bc409ca4b3e8", - "zh:2b881032b9aa9d227ac712f614056d050bcdcc67df0dc79e2b2cb76a197059ad", - "zh:38feb4787b4570335459ca75a55389df1a7570bdca8cdf5df4c2876afe3c14b4", - "zh:40f7e0aaef3b1f4c2ca2bb1189e3fe9af8c296da129423986d1d99ccc8cfb86c", - "zh:56b361f64f0f0df5c4f958ae2f0e6f8ba192f35b720b9d3ae1be068fabcf73d9", - "zh:5fadb5880cd31c2105f635ded92b9b16f918c1dd989627a4ce62c04939223909", - "zh:61fa0be9c14c8c4109cfb7be8d54a80c56d35dbae49d3231cddb59831e7e5a4d", - "zh:853774bf97fbc4a784d5af5a4ca0090848430781ae6cfc586adeb48f7c44af79", + "h1:wbtDfLeawmv6xVT1W0w0fctRCb4ABlaD3JTxwb1jXag=", + "zh:0d83ffb72fbd08986378204a7373d8c43b127049096eaf2765bfdd6b00ad9853", + "zh:7577d6edc67b1e8c2cf62fe6501192df1231d74125d90e51d570d586d95269c5", + "zh:9c669ded5d5affa4b2544952c4b6588dfed55260147d24ced02dca3a2829f328", + "zh:a404d46f2831f90633947ab5d57e19dbfe35b3704104ba6ec80bcf50b058acfd", + "zh:ae1caea1c936d459ceadf287bb5c5bd67b5e2a7819df6f5c4114b7305df7f822", + "zh:afb4f805477694a4b9dde86b268d2c0821711c8aab1c6088f5f992228c4c06fb", + "zh:b993b4a1de8a462643e78f4786789e44ce5064b332fee1cb0d6250ed085561b8", + "zh:c84b2c13fa3ea2c0aa7291243006d560ce480a5591294b9001ce3742fc9c5791", + "zh:c8966f69b7eccccb771704fd5335923692eccc9e0e90cb95d14538fe2e92a3b8", + "zh:d5fe68850d449b811e633a300b114d0617df6d450305e8251643b4d143dc855b", + "zh:ddebfd1e674ba336df09b1f27bbaa0e036c25b7a7087dc8081443f6e5954028b", ] } diff --git a/tf/infra.tf b/tf/infra.tf index 747da42..7513ed8 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -6,14 +6,15 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 2.70" + version = "~> 3.45" } } } locals { tags = { - Terraform = "Managed by Terraform" + Terraform = "Managed by Terraform" + BillingCategory = "Riju" } } @@ -171,6 +172,11 @@ resource "aws_instance" "server" { tags = merge(local.tags, { Name = "Riju server" }) + root_block_device { + tags = merge(local.tags, { + Name = "Riju server root volume" + }) + } } resource "aws_ebs_volume" "data" { From 1cd1338eb59390f79a853877948cd903fc70acd9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 13:13:29 -0700 Subject: [PATCH 040/104] Block public S3 access --- tf/infra.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tf/infra.tf b/tf/infra.tf index 7513ed8..bf81128 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -107,6 +107,15 @@ resource "aws_s3_bucket" "riju" { tags = local.tags } +resource "aws_s3_bucket_public_access_block" "riju" { + bucket = aws_s3_bucket.riju.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + resource "aws_s3_bucket_policy" "riju" { bucket = aws_s3_bucket.riju.id policy = data.aws_iam_policy_document.riju.json From a95f9b2d50d972ea82e7719471f8c6a8cad49f65 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 21:01:32 +0000 Subject: [PATCH 041/104] Fix a permissions error --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 0db51f8..85f2e2f 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,7 @@ IMAGE_HASH := -e RIJU_IMAGE_HASH="$$(docker inspect riju:$(LANG_TAG) | jq '.[0]. shell: # I= [L=] [E=1] [P1|P2=] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) + @mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) From d5277cbf67c9cb4033b6a4afb666152aa0e18617 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 21:08:30 +0000 Subject: [PATCH 042/104] Add security policy --- SECURITY.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..81537d6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,11 @@ +# Reporting a security issue + +Please contact me at +[radon.neon@gmail.com](mailto:radon.neon@gmail.com) if you find any +way to: + +* Take down Riju without using a large number of concurrent sessions. +* View or interfere with another user's session. +* Tamper with the Riju server. + +I will do my best to correct the vulnerability as soon as possible. From e5ec173f3cefb86c0db7629231540a99b547a76e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 21:10:32 +0000 Subject: [PATCH 043/104] Upgrade dependencies --- yarn.lock | 1842 +++++++++++++++++++++++++++-------------------------- 1 file changed, 940 insertions(+), 902 deletions(-) diff --git a/yarn.lock b/yarn.lock index 91022fc..3d37d81 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,382 +2,419 @@ # yarn lockfile v1 -"@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" - integrity sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw== +"@babel/code-frame@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.14.5.tgz#23b08d740e83f49c5e59945fbf1b43e80bbf4edb" + integrity sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM+vOtCS5ndmJicPJhKAwYRI6UfFw== dependencies: - "@babel/highlight" "^7.10.4" + "@babel/highlight" "^7.14.5" -"@babel/compat-data@^7.12.5", "@babel/compat-data@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.12.7.tgz#9329b4782a7d6bbd7eef57e11addf91ee3ef1e41" - integrity sha512-YaxPMGs/XIWtYqrdEOZOCPsVWfEoriXopnsz3/i7apYPXQ3698UFhS6dVT1KN5qOsWmVgw/FOrmQgpRaZayGsw== +"@babel/compat-data@^7.13.11", "@babel/compat-data@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.5.tgz#8ef4c18e58e801c5c95d3c1c0f2874a2680fadea" + integrity sha512-kixrYn4JwfAVPa0f2yfzc2AWti6WRRyO3XjWW5PJAvtE11qhSayrrcrEnee05KAtNaPC+EwehE8Qt1UedEVB8w== "@babel/core@^7.12.10": - version "7.12.10" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.10.tgz#b79a2e1b9f70ed3d84bbfb6d8c4ef825f606bccd" - integrity sha512-eTAlQKq65zHfkHZV0sIVODCPGVgoo1HdBlbSLi9CqOzuZanMv2ihzY+4paiKr1mH+XmYESMAmJ/dpZ68eN6d8w== + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.14.5.tgz#d281f46a9905f07d1b3bf71ead54d9c7d89cb1e3" + integrity sha512-RN/AwP2DJmQTZSfiDaD+JQQ/J99KsIpOCfBE5pL+5jJSt7nI3nYGoAXZu+ffYSQ029NLs2DstZb+eR81uuARgg== dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.10" - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helpers" "^7.12.5" - "@babel/parser" "^7.12.10" - "@babel/template" "^7.12.7" - "@babel/traverse" "^7.12.10" - "@babel/types" "^7.12.10" + "@babel/code-frame" "^7.14.5" + "@babel/generator" "^7.14.5" + "@babel/helper-compilation-targets" "^7.14.5" + "@babel/helper-module-transforms" "^7.14.5" + "@babel/helpers" "^7.14.5" + "@babel/parser" "^7.14.5" + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.5" + "@babel/types" "^7.14.5" convert-source-map "^1.7.0" debug "^4.1.0" - gensync "^1.0.0-beta.1" + gensync "^1.0.0-beta.2" json5 "^2.1.2" - lodash "^4.17.19" - semver "^5.4.1" + semver "^6.3.0" source-map "^0.5.0" -"@babel/generator@^7.12.10", "@babel/generator@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.11.tgz#98a7df7b8c358c9a37ab07a24056853016aba3af" - integrity sha512-Ggg6WPOJtSi8yYQvLVjG8F/TlpWDlKx0OpS4Kt+xMQPs5OaGYWy+v1A+1TvxI6sAMGZpKWWoAQ1DaeQbImlItA== +"@babel/generator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.14.5.tgz#848d7b9f031caca9d0cd0af01b063f226f52d785" + integrity sha512-y3rlP+/G25OIX3mYKKIOlQRcqj7YgrvHxOLbVmyLJ9bPmi5ttvUmpydVjcFjZphOktWuA7ovbx91ECloWTfjIA== dependencies: - "@babel/types" "^7.12.11" + "@babel/types" "^7.14.5" jsesc "^2.5.1" source-map "^0.5.0" -"@babel/helper-annotate-as-pure@^7.10.4": - version "7.12.10" - resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.10.tgz#54ab9b000e60a93644ce17b3f37d313aaf1d115d" - integrity sha512-XplmVbC1n+KY6jL8/fgLVXXUauDIB+lD5+GsQEh6F6GBF1dq1qy4DP4yXWzDKcoqXB3X58t61e85Fitoww4JVQ== +"@babel/helper-annotate-as-pure@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.14.5.tgz#7bf478ec3b71726d56a8ca5775b046fc29879e61" + integrity sha512-EivH9EgBIb+G8ij1B2jAwSH36WnGvkQSEC6CkX/6v6ZFlw5fVOHvsgGF4uiEHO2GzMvunZb6tDLQEQSdrdocrA== dependencies: - "@babel/types" "^7.12.10" + "@babel/types" "^7.14.5" -"@babel/helper-builder-binary-assignment-operator-visitor@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.10.4.tgz#bb0b75f31bf98cbf9ff143c1ae578b87274ae1a3" - integrity sha512-L0zGlFrGWZK4PbT8AszSfLTM5sDU1+Az/En9VrdT8/LmEiJt4zXt+Jve9DCAnQcbqDhCI+29y/L93mrDzddCcg== +"@babel/helper-builder-binary-assignment-operator-visitor@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.14.5.tgz#b939b43f8c37765443a19ae74ad8b15978e0a191" + integrity sha512-YTA/Twn0vBXDVGJuAX6PwW7x5zQei1luDDo2Pl6q1qZ7hVNl0RZrhHCQG/ArGpR29Vl7ETiB8eJyrvpuRp300w== dependencies: - "@babel/helper-explode-assignable-expression" "^7.10.4" - "@babel/types" "^7.10.4" + "@babel/helper-explode-assignable-expression" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/helper-compilation-targets@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.12.5.tgz#cb470c76198db6a24e9dbc8987275631e5d29831" - integrity sha512-+qH6NrscMolUlzOYngSBMIOQpKUGPPsc61Bu5W10mg84LxZ7cmvnBHzARKbDoFxVvqqAbj6Tg6N7bSrWSPXMyw== +"@babel/helper-compilation-targets@^7.13.0", "@babel/helper-compilation-targets@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.14.5.tgz#7a99c5d0967911e972fe2c3411f7d5b498498ecf" + integrity sha512-v+QtZqXEiOnpO6EYvlImB6zCD2Lel06RzOPzmkz/D/XgQiUu3C/Jb1LOqSt/AIA34TYi/Q+KlT8vTQrgdxkbLw== dependencies: - "@babel/compat-data" "^7.12.5" - "@babel/helper-validator-option" "^7.12.1" - browserslist "^4.14.5" - semver "^5.5.0" + "@babel/compat-data" "^7.14.5" + "@babel/helper-validator-option" "^7.14.5" + browserslist "^4.16.6" + semver "^6.3.0" -"@babel/helper-create-class-features-plugin@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.12.1.tgz#3c45998f431edd4a9214c5f1d3ad1448a6137f6e" - integrity sha512-hkL++rWeta/OVOBTRJc9a5Azh5mt5WgZUGAKMD8JM141YsE08K//bp1unBBieO6rUKkIPyUE0USQ30jAy3Sk1w== +"@babel/helper-create-class-features-plugin@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.14.5.tgz#8842ec495516dd1ed8f6c572be92ba78b1e9beef" + integrity sha512-Uq9z2e7ZtcnDMirRqAGLRaLwJn+Lrh388v5ETrR3pALJnElVh2zqQmdbz4W2RUJYohAPh2mtyPUgyMHMzXMncQ== dependencies: - "@babel/helper-function-name" "^7.10.4" - "@babel/helper-member-expression-to-functions" "^7.12.1" - "@babel/helper-optimise-call-expression" "^7.10.4" - "@babel/helper-replace-supers" "^7.12.1" - "@babel/helper-split-export-declaration" "^7.10.4" + "@babel/helper-annotate-as-pure" "^7.14.5" + "@babel/helper-function-name" "^7.14.5" + "@babel/helper-member-expression-to-functions" "^7.14.5" + "@babel/helper-optimise-call-expression" "^7.14.5" + "@babel/helper-replace-supers" "^7.14.5" + "@babel/helper-split-export-declaration" "^7.14.5" -"@babel/helper-create-regexp-features-plugin@^7.12.1": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.7.tgz#2084172e95443fa0a09214ba1bb328f9aea1278f" - integrity sha512-idnutvQPdpbduutvi3JVfEgcVIHooQnhvhx0Nk9isOINOIGYkZea1Pk2JlJRiUnMefrlvr0vkByATBY/mB4vjQ== +"@babel/helper-create-regexp-features-plugin@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.14.5.tgz#c7d5ac5e9cf621c26057722fb7a8a4c5889358c4" + integrity sha512-TLawwqpOErY2HhWbGJ2nZT5wSkR192QpN+nBg1THfBfftrlvOh+WbhrxXCH4q4xJ9Gl16BGPR/48JA+Ryiho/A== dependencies: - "@babel/helper-annotate-as-pure" "^7.10.4" + "@babel/helper-annotate-as-pure" "^7.14.5" regexpu-core "^4.7.1" -"@babel/helper-define-map@^7.10.4": - version "7.10.5" - resolved "https://registry.yarnpkg.com/@babel/helper-define-map/-/helper-define-map-7.10.5.tgz#b53c10db78a640800152692b13393147acb9bb30" - integrity sha512-fMw4kgFB720aQFXSVaXr79pjjcW5puTCM16+rECJ/plGS+zByelE8l9nCpV1GibxTnFVmUuYG9U8wYfQHdzOEQ== +"@babel/helper-define-polyfill-provider@^0.2.2": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.2.3.tgz#0525edec5094653a282688d34d846e4c75e9c0b6" + integrity sha512-RH3QDAfRMzj7+0Nqu5oqgO5q9mFtQEVvCRsi8qCEfzLR9p2BHfn5FzhSB2oj1fF7I2+DcTORkYaQ6aTR9Cofew== dependencies: - "@babel/helper-function-name" "^7.10.4" - "@babel/types" "^7.10.5" - lodash "^4.17.19" + "@babel/helper-compilation-targets" "^7.13.0" + "@babel/helper-module-imports" "^7.12.13" + "@babel/helper-plugin-utils" "^7.13.0" + "@babel/traverse" "^7.13.0" + debug "^4.1.1" + lodash.debounce "^4.0.8" + resolve "^1.14.2" + semver "^6.1.2" -"@babel/helper-explode-assignable-expression@^7.10.4": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.12.1.tgz#8006a466695c4ad86a2a5f2fb15b5f2c31ad5633" - integrity sha512-dmUwH8XmlrUpVqgtZ737tK88v07l840z9j3OEhCLwKTkjlvKpfqXVIZ0wpK3aeOxspwGrf/5AP5qLx4rO3w5rA== +"@babel/helper-explode-assignable-expression@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.14.5.tgz#8aa72e708205c7bb643e45c73b4386cdf2a1f645" + integrity sha512-Htb24gnGJdIGT4vnRKMdoXiOIlqOLmdiUYpAQ0mYfgVT/GDm8GOYhgi4GL+hMKrkiPRohO4ts34ELFsGAPQLDQ== dependencies: - "@babel/types" "^7.12.1" + "@babel/types" "^7.14.5" -"@babel/helper-function-name@^7.10.4", "@babel/helper-function-name@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.12.11.tgz#1fd7738aee5dcf53c3ecff24f1da9c511ec47b42" - integrity sha512-AtQKjtYNolKNi6nNNVLQ27CP6D9oFR6bq/HPYSizlzbp7uC1M59XJe8L+0uXjbIaZaUJF99ruHqVGiKXU/7ybA== +"@babel/helper-function-name@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.14.5.tgz#89e2c474972f15d8e233b52ee8c480e2cfcd50c4" + integrity sha512-Gjna0AsXWfFvrAuX+VKcN/aNNWonizBj39yGwUzVDVTlMYJMK2Wp6xdpy72mfArFq5uK+NOuexfzZlzI1z9+AQ== dependencies: - "@babel/helper-get-function-arity" "^7.12.10" - "@babel/template" "^7.12.7" - "@babel/types" "^7.12.11" + "@babel/helper-get-function-arity" "^7.14.5" + "@babel/template" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/helper-get-function-arity@^7.12.10": - version "7.12.10" - resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.10.tgz#b158817a3165b5faa2047825dfa61970ddcc16cf" - integrity sha512-mm0n5BPjR06wh9mPQaDdXWDoll/j5UpCAPl1x8fS71GHm7HA6Ua2V4ylG1Ju8lvcTOietbPNNPaSilKj+pj+Ag== +"@babel/helper-get-function-arity@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz#25fbfa579b0937eee1f3b805ece4ce398c431815" + integrity sha512-I1Db4Shst5lewOM4V+ZKJzQ0JGGaZ6VY1jYvMghRjqs6DWgxLCIyFt30GlnKkfUeFLpJt2vzbMVEXVSXlIFYUg== dependencies: - "@babel/types" "^7.12.10" + "@babel/types" "^7.14.5" -"@babel/helper-hoist-variables@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.4.tgz#d49b001d1d5a68ca5e6604dda01a6297f7c9381e" - integrity sha512-wljroF5PgCk2juF69kanHVs6vrLwIPNp6DLD+Lrl3hoQ3PpPPikaDRNFA+0t81NOoMt2DL6WW/mdU8k4k6ZzuA== +"@babel/helper-hoist-variables@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.14.5.tgz#e0dd27c33a78e577d7c8884916a3e7ef1f7c7f8d" + integrity sha512-R1PXiz31Uc0Vxy4OEOm07x0oSjKAdPPCh3tPivn/Eo8cvz6gveAeuyUUPB21Hoiif0uoPQSSdhIPS3352nvdyQ== dependencies: - "@babel/types" "^7.10.4" + "@babel/types" "^7.14.5" -"@babel/helper-member-expression-to-functions@^7.12.1", "@babel/helper-member-expression-to-functions@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.7.tgz#aa77bd0396ec8114e5e30787efa78599d874a855" - integrity sha512-DCsuPyeWxeHgh1Dus7APn7iza42i/qXqiFPWyBDdOFtvS581JQePsc1F/nD+fHrcswhLlRc2UpYS1NwERxZhHw== +"@babel/helper-member-expression-to-functions@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.5.tgz#d5c70e4ad13b402c95156c7a53568f504e2fb7b8" + integrity sha512-UxUeEYPrqH1Q/k0yRku1JE7dyfyehNwT6SVkMHvYvPDv4+uu627VXBckVj891BO8ruKBkiDoGnZf4qPDD8abDQ== dependencies: - "@babel/types" "^7.12.7" + "@babel/types" "^7.14.5" -"@babel/helper-module-imports@^7.12.1", "@babel/helper-module-imports@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz#1bfc0229f794988f76ed0a4d4e90860850b54dfb" - integrity sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA== +"@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.14.5.tgz#6d1a44df6a38c957aa7c312da076429f11b422f3" + integrity sha512-SwrNHu5QWS84XlHwGYPDtCxcA0hrSlL2yhWYLgeOc0w7ccOl2qv4s/nARI0aYZW+bSwAL5CukeXA47B/1NKcnQ== dependencies: - "@babel/types" "^7.12.5" + "@babel/types" "^7.14.5" -"@babel/helper-module-transforms@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.12.1.tgz#7954fec71f5b32c48e4b303b437c34453fd7247c" - integrity sha512-QQzehgFAZ2bbISiCpmVGfiGux8YVFXQ0abBic2Envhej22DVXV9nCFaS5hIQbkyo1AdGb+gNME2TSh3hYJVV/w== +"@babel/helper-module-transforms@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.14.5.tgz#7de42f10d789b423eb902ebd24031ca77cb1e10e" + integrity sha512-iXpX4KW8LVODuAieD7MzhNjmM6dzYY5tfRqT+R9HDXWl0jPn/djKmA+G9s/2C2T9zggw5tK1QNqZ70USfedOwA== dependencies: - "@babel/helper-module-imports" "^7.12.1" - "@babel/helper-replace-supers" "^7.12.1" - "@babel/helper-simple-access" "^7.12.1" - "@babel/helper-split-export-declaration" "^7.11.0" - "@babel/helper-validator-identifier" "^7.10.4" - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.12.1" - "@babel/types" "^7.12.1" - lodash "^4.17.19" + "@babel/helper-module-imports" "^7.14.5" + "@babel/helper-replace-supers" "^7.14.5" + "@babel/helper-simple-access" "^7.14.5" + "@babel/helper-split-export-declaration" "^7.14.5" + "@babel/helper-validator-identifier" "^7.14.5" + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/helper-optimise-call-expression@^7.10.4", "@babel/helper-optimise-call-expression@^7.12.10": - version "7.12.10" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.10.tgz#94ca4e306ee11a7dd6e9f42823e2ac6b49881e2d" - integrity sha512-4tpbU0SrSTjjt65UMWSrUOPZTsgvPgGG4S8QSTNHacKzpS51IVWGDj0yCwyeZND/i+LSN2g/O63jEXEWm49sYQ== +"@babel/helper-optimise-call-expression@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.14.5.tgz#f27395a8619e0665b3f0364cddb41c25d71b499c" + integrity sha512-IqiLIrODUOdnPU9/F8ib1Fx2ohlgDhxnIDU7OEVi+kAbEZcyiF7BLU8W6PfvPi9LzztjS7kcbzbmL7oG8kD6VA== dependencies: - "@babel/types" "^7.12.10" + "@babel/types" "^7.14.5" -"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz#2f75a831269d4f677de49986dff59927533cf375" - integrity sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg== +"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.13.0", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.14.5.tgz#5ac822ce97eec46741ab70a517971e443a70c5a9" + integrity sha512-/37qQCE3K0vvZKwoK4XU/irIJQdIfCJuhU5eKnNxpFDsOkgFaUAwbv+RYw6eYgsC0E4hS7r5KqGULUogqui0fQ== -"@babel/helper-remap-async-to-generator@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.12.1.tgz#8c4dbbf916314f6047dc05e6a2217074238347fd" - integrity sha512-9d0KQCRM8clMPcDwo8SevNs+/9a8yWVVmaE80FGJcEP8N1qToREmWEGnBn8BUlJhYRFz6fqxeRL1sl5Ogsed7A== +"@babel/helper-remap-async-to-generator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.14.5.tgz#51439c913612958f54a987a4ffc9ee587a2045d6" + integrity sha512-rLQKdQU+HYlxBwQIj8dk4/0ENOUEhA/Z0l4hN8BexpvmSMN9oA9EagjnhnDpNsRdWCfjwa4mn/HyBXO9yhQP6A== dependencies: - "@babel/helper-annotate-as-pure" "^7.10.4" - "@babel/helper-wrap-function" "^7.10.4" - "@babel/types" "^7.12.1" + "@babel/helper-annotate-as-pure" "^7.14.5" + "@babel/helper-wrap-function" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/helper-replace-supers@^7.12.1": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.12.11.tgz#ea511658fc66c7908f923106dd88e08d1997d60d" - integrity sha512-q+w1cqmhL7R0FNzth/PLLp2N+scXEK/L2AHbXUyydxp828F4FEa5WcVoqui9vFRiHDQErj9Zof8azP32uGVTRA== +"@babel/helper-replace-supers@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.14.5.tgz#0ecc0b03c41cd567b4024ea016134c28414abb94" + integrity sha512-3i1Qe9/8x/hCHINujn+iuHy+mMRLoc77b2nI9TB0zjH1hvn9qGlXjWlggdwUcju36PkPCy/lpM7LLUdcTyH4Ow== dependencies: - "@babel/helper-member-expression-to-functions" "^7.12.7" - "@babel/helper-optimise-call-expression" "^7.12.10" - "@babel/traverse" "^7.12.10" - "@babel/types" "^7.12.11" + "@babel/helper-member-expression-to-functions" "^7.14.5" + "@babel/helper-optimise-call-expression" "^7.14.5" + "@babel/traverse" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/helper-simple-access@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.12.1.tgz#32427e5aa61547d38eb1e6eaf5fd1426fdad9136" - integrity sha512-OxBp7pMrjVewSSC8fXDFrHrBcJATOOFssZwv16F3/6Xtc138GHybBfPbm9kfiqQHKhYQrlamWILwlDCeyMFEaA== +"@babel/helper-simple-access@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.14.5.tgz#66ea85cf53ba0b4e588ba77fc813f53abcaa41c4" + integrity sha512-nfBN9xvmCt6nrMZjfhkl7i0oTV3yxR4/FztsbOASyTvVcoYd0TRHh7eMLdlEcCqobydC0LAF3LtC92Iwxo0wyw== dependencies: - "@babel/types" "^7.12.1" + "@babel/types" "^7.14.5" -"@babel/helper-skip-transparent-expression-wrappers@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.12.1.tgz#462dc63a7e435ade8468385c63d2b84cce4b3cbf" - integrity sha512-Mf5AUuhG1/OCChOJ/HcADmvcHM42WJockombn8ATJG3OnyiSxBK/Mm5x78BQWvmtXZKHgbjdGL2kin/HOLlZGA== +"@babel/helper-skip-transparent-expression-wrappers@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.14.5.tgz#96f486ac050ca9f44b009fbe5b7d394cab3a0ee4" + integrity sha512-dmqZB7mrb94PZSAOYtr+ZN5qt5owZIAgqtoTuqiFbHFtxgEcmQlRJVI+bO++fciBunXtB6MK7HrzrfcAzIz2NQ== dependencies: - "@babel/types" "^7.12.1" + "@babel/types" "^7.14.5" -"@babel/helper-split-export-declaration@^7.10.4", "@babel/helper-split-export-declaration@^7.11.0", "@babel/helper-split-export-declaration@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.11.tgz#1b4cc424458643c47d37022223da33d76ea4603a" - integrity sha512-LsIVN8j48gHgwzfocYUSkO/hjYAOJqlpJEc7tGXcIm4cubjVUf8LGW6eWRyxEu7gA25q02p0rQUWoCI33HNS5g== +"@babel/helper-split-export-declaration@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.14.5.tgz#22b23a54ef51c2b7605d851930c1976dd0bc693a" + integrity sha512-hprxVPu6e5Kdp2puZUmvOGjaLv9TCe58E/Fl6hRq4YiVQxIcNvuq6uTM2r1mT/oPskuS9CgR+I94sqAYv0NGKA== dependencies: - "@babel/types" "^7.12.11" + "@babel/types" "^7.14.5" -"@babel/helper-validator-identifier@^7.10.4", "@babel/helper-validator-identifier@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" - integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== +"@babel/helper-validator-identifier@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8" + integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg== -"@babel/helper-validator-option@^7.12.1", "@babel/helper-validator-option@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.12.11.tgz#d66cb8b7a3e7fe4c6962b32020a131ecf0847f4f" - integrity sha512-TBFCyj939mFSdeX7U7DDj32WtzYY7fDcalgq8v3fBZMNOJQNn7nOYzMaUCiPxPYfCup69mtIpqlKgMZLvQ8Xhw== +"@babel/helper-validator-option@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz#6e72a1fff18d5dfcb878e1e62f1a021c4b72d5a3" + integrity sha512-OX8D5eeX4XwcroVW45NMvoYaIuFI+GQpA2a8Gi+X/U/cDUIRsV37qQfF905F0htTRCREQIB4KqPeaveRJUl3Ow== -"@babel/helper-wrap-function@^7.10.4": - version "7.12.3" - resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.12.3.tgz#3332339fc4d1fbbf1c27d7958c27d34708e990d9" - integrity sha512-Cvb8IuJDln3rs6tzjW3Y8UeelAOdnpB8xtQ4sme2MSZ9wOxrbThporC0y/EtE16VAtoyEfLM404Xr1e0OOp+ow== +"@babel/helper-wrap-function@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.14.5.tgz#5919d115bf0fe328b8a5d63bcb610f51601f2bff" + integrity sha512-YEdjTCq+LNuNS1WfxsDCNpgXkJaIyqco6DAelTUjT4f2KIWC1nBcaCaSdHTBqQVLnTBexBcVcFhLSU1KnYuePQ== dependencies: - "@babel/helper-function-name" "^7.10.4" - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.10.4" - "@babel/types" "^7.10.4" + "@babel/helper-function-name" "^7.14.5" + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/helpers@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.12.5.tgz#1a1ba4a768d9b58310eda516c449913fe647116e" - integrity sha512-lgKGMQlKqA8meJqKsW6rUnc4MdUk35Ln0ATDqdM1a/UpARODdI4j5Y5lVfUScnSNkJcdCRAaWkspykNoFg9sJA== +"@babel/helpers@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.14.5.tgz#4870f8d9a6fdbbd65e5674a3558b4ff7fef0d9b2" + integrity sha512-xtcWOuN9VL6nApgVHtq3PPcQv5qFBJzoSZzJ/2c0QK/IP/gxVcoWSNQwFEGvmbQsuS9rhYqjILDGGXcTkA705Q== dependencies: - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.12.5" - "@babel/types" "^7.12.5" + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/highlight@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.4.tgz#7d1bdfd65753538fabe6c38596cdb76d9ac60143" - integrity sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA== +"@babel/highlight@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.14.5.tgz#6861a52f03966405001f6aa534a01a24d99e8cd9" + integrity sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg== dependencies: - "@babel/helper-validator-identifier" "^7.10.4" + "@babel/helper-validator-identifier" "^7.14.5" chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.12.10", "@babel/parser@^7.12.11", "@babel/parser@^7.12.7": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.11.tgz#9ce3595bcd74bc5c466905e86c535b8b25011e79" - integrity sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg== +"@babel/parser@^7.13.11", "@babel/parser@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.5.tgz#4cd2f346261061b2518873ffecdf1612cb032829" + integrity sha512-TM8C+xtH/9n1qzX+JNHi7AN2zHMTiPUtspO0ZdHflW8KaskkALhMmuMHb4bCmNdv9VAPzJX3/bXqkVLnAvsPfg== -"@babel/parser@^7.13.11": - version "7.13.11" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.11.tgz#f93ebfc99d21c1772afbbaa153f47e7ce2f50b88" - integrity sha512-PhuoqeHoO9fc4ffMEVk4qb/w/s2iOSWohvbHxLtxui0eBg3Lg5gN1U8wp1V1u61hOWkPQJJyJzGH6Y+grwkq8Q== - -"@babel/plugin-proposal-async-generator-functions@^7.12.1": - version "7.12.12" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.12.tgz#04b8f24fd4532008ab4e79f788468fd5a8476566" - integrity sha512-nrz9y0a4xmUrRq51bYkWJIO5SBZyG2ys2qinHsN0zHDHVsUaModrkpyWWWXfGqYQmOL3x9sQIcTNN/pBGpo09A== +"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.14.5.tgz#4b467302e1548ed3b1be43beae2cc9cf45e0bb7e" + integrity sha512-ZoJS2XCKPBfTmL122iP6NM9dOg+d4lc9fFk3zxc8iDjvt8Pk4+TlsHSKhIPf6X+L5ORCdBzqMZDjL/WHj7WknQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-remap-async-to-generator" "^7.12.1" - "@babel/plugin-syntax-async-generators" "^7.8.0" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-skip-transparent-expression-wrappers" "^7.14.5" + "@babel/plugin-proposal-optional-chaining" "^7.14.5" -"@babel/plugin-proposal-class-properties@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.12.1.tgz#a082ff541f2a29a4821065b8add9346c0c16e5de" - integrity sha512-cKp3dlQsFsEs5CWKnN7BnSHOd0EOW8EKpEjkoz1pO2E5KzIDNV9Ros1b0CnmbVgAGXJubOYVBOGCT1OmJwOI7w== +"@babel/plugin-proposal-async-generator-functions@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.14.5.tgz#4024990e3dd74181f4f426ea657769ff49a2df39" + integrity sha512-tbD/CG3l43FIXxmu4a7RBe4zH7MLJ+S/lFowPFO7HetS2hyOZ/0nnnznegDuzFzfkyQYTxqdTH/hKmuBngaDAA== dependencies: - "@babel/helper-create-class-features-plugin" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-remap-async-to-generator" "^7.14.5" + "@babel/plugin-syntax-async-generators" "^7.8.4" -"@babel/plugin-proposal-dynamic-import@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.12.1.tgz#43eb5c2a3487ecd98c5c8ea8b5fdb69a2749b2dc" - integrity sha512-a4rhUSZFuq5W8/OO8H7BL5zspjnc1FLd9hlOxIK/f7qG4a0qsqk8uvF/ywgBA8/OmjsapjpvaEOYItfGG1qIvQ== +"@babel/plugin-proposal-class-properties@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.14.5.tgz#40d1ee140c5b1e31a350f4f5eed945096559b42e" + integrity sha512-q/PLpv5Ko4dVc1LYMpCY7RVAAO4uk55qPwrIuJ5QJ8c6cVuAmhu7I/49JOppXL6gXf7ZHzpRVEUZdYoPLM04Gg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-dynamic-import" "^7.8.0" + "@babel/helper-create-class-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-proposal-export-namespace-from@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.12.1.tgz#8b9b8f376b2d88f5dd774e4d24a5cc2e3679b6d4" - integrity sha512-6CThGf0irEkzujYS5LQcjBx8j/4aQGiVv7J9+2f7pGfxqyKh3WnmVJYW3hdrQjyksErMGBPQrCnHfOtna+WLbw== +"@babel/plugin-proposal-class-static-block@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.14.5.tgz#158e9e10d449c3849ef3ecde94a03d9f1841b681" + integrity sha512-KBAH5ksEnYHCegqseI5N9skTdxgJdmDoAOc0uXa+4QMYKeZD0w5IARh4FMlTNtaHhbB8v+KzMdTgxMMzsIy6Yg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-create-class-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-class-static-block" "^7.14.5" + +"@babel/plugin-proposal-dynamic-import@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.14.5.tgz#0c6617df461c0c1f8fff3b47cd59772360101d2c" + integrity sha512-ExjiNYc3HDN5PXJx+bwC50GIx/KKanX2HiggnIUAYedbARdImiCU4RhhHfdf0Kd7JNXGpsBBBCOm+bBVy3Gb0g== + dependencies: + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-dynamic-import" "^7.8.3" + +"@babel/plugin-proposal-export-namespace-from@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.14.5.tgz#dbad244310ce6ccd083072167d8cea83a52faf76" + integrity sha512-g5POA32bXPMmSBu5Dx/iZGLGnKmKPc5AiY7qfZgurzrCYgIztDlHFbznSNCoQuv57YQLnQfaDi7dxCtLDIdXdA== + dependencies: + "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-export-namespace-from" "^7.8.3" -"@babel/plugin-proposal-json-strings@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.12.1.tgz#d45423b517714eedd5621a9dfdc03fa9f4eb241c" - integrity sha512-GoLDUi6U9ZLzlSda2Df++VSqDJg3CG+dR0+iWsv6XRw1rEq+zwt4DirM9yrxW6XWaTpmai1cWJLMfM8qQJf+yw== +"@babel/plugin-proposal-json-strings@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.14.5.tgz#38de60db362e83a3d8c944ac858ddf9f0c2239eb" + integrity sha512-NSq2fczJYKVRIsUJyNxrVUMhB27zb7N7pOFGQOhBKJrChbGcgEAqyZrmZswkPk18VMurEeJAaICbfm57vUeTbQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-json-strings" "^7.8.0" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-json-strings" "^7.8.3" -"@babel/plugin-proposal-logical-assignment-operators@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.12.1.tgz#f2c490d36e1b3c9659241034a5d2cd50263a2751" - integrity sha512-k8ZmVv0JU+4gcUGeCDZOGd0lCIamU/sMtIiX3UWnUc5yzgq6YUGyEolNYD+MLYKfSzgECPcqetVcJP9Afe/aCA== +"@babel/plugin-proposal-logical-assignment-operators@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.14.5.tgz#6e6229c2a99b02ab2915f82571e0cc646a40c738" + integrity sha512-YGn2AvZAo9TwyhlLvCCWxD90Xq8xJ4aSgaX3G5D/8DW94L8aaT+dS5cSP+Z06+rCJERGSr9GxMBZ601xoc2taw== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" -"@babel/plugin-proposal-nullish-coalescing-operator@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.12.1.tgz#3ed4fff31c015e7f3f1467f190dbe545cd7b046c" - integrity sha512-nZY0ESiaQDI1y96+jk6VxMOaL4LPo/QDHBqL+SF3/vl6dHkTwHlOI8L4ZwuRBHgakRBw5zsVylel7QPbbGuYgg== +"@babel/plugin-proposal-nullish-coalescing-operator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.14.5.tgz#ee38589ce00e2cc59b299ec3ea406fcd3a0fdaf6" + integrity sha512-gun/SOnMqjSb98Nkaq2rTKMwervfdAoz6NphdY0vTfuzMfryj+tDGb2n6UkDKwez+Y8PZDhE3D143v6Gepp4Hg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" -"@babel/plugin-proposal-numeric-separator@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.7.tgz#8bf253de8139099fea193b297d23a9d406ef056b" - integrity sha512-8c+uy0qmnRTeukiGsjLGy6uVs/TFjJchGXUeBqlG4VWYOdJWkhhVPdQ3uHwbmalfJwv2JsV0qffXP4asRfL2SQ== +"@babel/plugin-proposal-numeric-separator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.14.5.tgz#83631bf33d9a51df184c2102a069ac0c58c05f18" + integrity sha512-yiclALKe0vyZRZE0pS6RXgjUOt87GWv6FYa5zqj15PvhOGFO69R5DusPlgK/1K5dVnCtegTiWu9UaBSrLLJJBg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" "@babel/plugin-syntax-numeric-separator" "^7.10.4" -"@babel/plugin-proposal-object-rest-spread@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz#def9bd03cea0f9b72283dac0ec22d289c7691069" - integrity sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA== +"@babel/plugin-proposal-object-rest-spread@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.14.5.tgz#e581d5ccdfa187ea6ed73f56c6a21c1580b90fbf" + integrity sha512-VzMyY6PWNPPT3pxc5hi9LloKNr4SSrVCg7Yr6aZpW4Ym07r7KqSU/QXYwjXLVxqwSv0t/XSXkFoKBPUkZ8vb2A== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-object-rest-spread" "^7.8.0" - "@babel/plugin-transform-parameters" "^7.12.1" + "@babel/compat-data" "^7.14.5" + "@babel/helper-compilation-targets" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-object-rest-spread" "^7.8.3" + "@babel/plugin-transform-parameters" "^7.14.5" -"@babel/plugin-proposal-optional-catch-binding@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.12.1.tgz#ccc2421af64d3aae50b558a71cede929a5ab2942" - integrity sha512-hFvIjgprh9mMw5v42sJWLI1lzU5L2sznP805zeT6rySVRA0Y18StRhDqhSxlap0oVgItRsB6WSROp4YnJTJz0g== +"@babel/plugin-proposal-optional-catch-binding@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.14.5.tgz#939dd6eddeff3a67fdf7b3f044b5347262598c3c" + integrity sha512-3Oyiixm0ur7bzO5ybNcZFlmVsygSIQgdOa7cTfOYCMY+wEPAYhZAJxi3mixKFCTCKUhQXuCTtQ1MzrpL3WT8ZQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" -"@babel/plugin-proposal-optional-chaining@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.7.tgz#e02f0ea1b5dc59d401ec16fb824679f683d3303c" - integrity sha512-4ovylXZ0PWmwoOvhU2vhnzVNnm88/Sm9nx7V8BPgMvAzn5zDou3/Awy0EjglyubVHasJj+XCEkr/r1X3P5elCA== +"@babel/plugin-proposal-optional-chaining@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.14.5.tgz#fa83651e60a360e3f13797eef00b8d519695b603" + integrity sha512-ycz+VOzo2UbWNI1rQXxIuMOzrDdHGrI23fRiz/Si2R4kv2XZQ1BK8ccdHwehMKBlcH/joGW/tzrUmo67gbJHlQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1" - "@babel/plugin-syntax-optional-chaining" "^7.8.0" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-skip-transparent-expression-wrappers" "^7.14.5" + "@babel/plugin-syntax-optional-chaining" "^7.8.3" -"@babel/plugin-proposal-private-methods@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.12.1.tgz#86814f6e7a21374c980c10d38b4493e703f4a389" - integrity sha512-mwZ1phvH7/NHK6Kf8LP7MYDogGV+DKB1mryFOEwx5EBNQrosvIczzZFTUmWaeujd5xT6G1ELYWUz3CutMhjE1w== +"@babel/plugin-proposal-private-methods@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.14.5.tgz#37446495996b2945f30f5be5b60d5e2aa4f5792d" + integrity sha512-838DkdUA1u+QTCplatfq4B7+1lnDa/+QMI89x5WZHBcnNv+47N8QEj2k9I2MUU9xIv8XJ4XvPCviM/Dj7Uwt9g== dependencies: - "@babel/helper-create-class-features-plugin" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-create-class-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-proposal-unicode-property-regex@^7.12.1", "@babel/plugin-proposal-unicode-property-regex@^7.4.4": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.12.1.tgz#2a183958d417765b9eae334f47758e5d6a82e072" - integrity sha512-MYq+l+PvHuw/rKUz1at/vb6nCnQ2gmJBNaM62z0OgH7B2W1D9pvkpYtlti9bGtizNIU1K3zm4bZF9F91efVY0w== +"@babel/plugin-proposal-private-property-in-object@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.14.5.tgz#9f65a4d0493a940b4c01f8aa9d3f1894a587f636" + integrity sha512-62EyfyA3WA0mZiF2e2IV9mc9Ghwxcg8YTu8BS4Wss4Y3PY725OmS9M0qLORbJwLqFtGh+jiE4wAmocK2CTUK2Q== dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-annotate-as-pure" "^7.14.5" + "@babel/helper-create-class-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/plugin-syntax-private-property-in-object" "^7.14.5" -"@babel/plugin-syntax-async-generators@^7.8.0": +"@babel/plugin-proposal-unicode-property-regex@^7.14.5", "@babel/plugin-proposal-unicode-property-regex@^7.4.4": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.14.5.tgz#0f95ee0e757a5d647f378daa0eca7e93faa8bbe8" + integrity sha512-6axIeOU5LnY471KenAB9vI8I5j7NQ2d652hIYwVyRfgaZT5UpiqFKCuVXCDMSrU+3VFafnu2c5m3lrWIlr6A5Q== + dependencies: + "@babel/helper-create-regexp-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + +"@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw== dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-class-properties@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.1.tgz#bcb297c5366e79bebadef509549cd93b04f19978" - integrity sha512-U40A76x5gTwmESz+qiqssqmeEsKvcSyvtgktrm0uzcARAmM9I1jR221f6Oq+GmHrcD+LvZDag1UTOTe2fL3TeA== +"@babel/plugin-syntax-class-properties@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10" + integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.12.13" -"@babel/plugin-syntax-dynamic-import@^7.8.0": +"@babel/plugin-syntax-class-static-block@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz#195df89b146b4b78b3bf897fd7a257c84659d406" + integrity sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw== + dependencies: + "@babel/helper-plugin-utils" "^7.14.5" + +"@babel/plugin-syntax-dynamic-import@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz#62bf98b2da3cd21d626154fc96ee5b3cb68eacb3" integrity sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ== @@ -391,7 +428,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.3" -"@babel/plugin-syntax-json-strings@^7.8.0": +"@babel/plugin-syntax-json-strings@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a" integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA== @@ -405,7 +442,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.4" -"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.0": +"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9" integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ== @@ -419,356 +456,369 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.4" -"@babel/plugin-syntax-object-rest-spread@^7.8.0": +"@babel/plugin-syntax-object-rest-spread@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871" integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-optional-catch-binding@^7.8.0": +"@babel/plugin-syntax-optional-catch-binding@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1" integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q== dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-optional-chaining@^7.8.0": +"@babel/plugin-syntax-optional-chaining@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a" integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg== dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-top-level-await@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" - integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== +"@babel/plugin-syntax-private-property-in-object@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz#0dc6671ec0ea22b6e94a1114f857970cd39de1ad" + integrity sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-arrow-functions@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.12.1.tgz#8083ffc86ac8e777fbe24b5967c4b2521f3cb2b3" - integrity sha512-5QB50qyN44fzzz4/qxDPQMBCTHgxg3n0xRBLJUmBlLoU/sFvxVWGZF/ZUfMVDQuJUKXaBhbupxIzIfZ6Fwk/0A== +"@babel/plugin-syntax-top-level-await@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz#c1cfdadc35a646240001f06138247b741c34d94c" + integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-async-to-generator@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.12.1.tgz#3849a49cc2a22e9743cbd6b52926d30337229af1" - integrity sha512-SDtqoEcarK1DFlRJ1hHRY5HvJUj5kX4qmtpMAm2QnhOlyuMC4TMdCRgW6WXpv93rZeYNeLP22y8Aq2dbcDRM1A== +"@babel/plugin-transform-arrow-functions@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.14.5.tgz#f7187d9588a768dd080bf4c9ffe117ea62f7862a" + integrity sha512-KOnO0l4+tD5IfOdi4x8C1XmEIRWUjNRV8wc6K2vz/3e8yAOoZZvsRXRRIF/yo/MAOFb4QjtAw9xSxMXbSMRy8A== dependencies: - "@babel/helper-module-imports" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-remap-async-to-generator" "^7.12.1" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-block-scoped-functions@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.12.1.tgz#f2a1a365bde2b7112e0a6ded9067fdd7c07905d9" - integrity sha512-5OpxfuYnSgPalRpo8EWGPzIYf0lHBWORCkj5M0oLBwHdlux9Ri36QqGW3/LR13RSVOAoUUMzoPI/jpE4ABcHoA== +"@babel/plugin-transform-async-to-generator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.14.5.tgz#72c789084d8f2094acb945633943ef8443d39e67" + integrity sha512-szkbzQ0mNk0rpu76fzDdqSyPu0MuvpXgC+6rz5rpMb5OIRxdmHfQxrktL8CYolL2d8luMCZTR0DpIMIdL27IjA== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-module-imports" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-remap-async-to-generator" "^7.14.5" -"@babel/plugin-transform-block-scoping@^7.12.11": - version "7.12.12" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.12.12.tgz#d93a567a152c22aea3b1929bb118d1d0a175cdca" - integrity sha512-VOEPQ/ExOVqbukuP7BYJtI5ZxxsmegTwzZ04j1aF0dkSypGo9XpDHuOrABsJu+ie+penpSJheDJ11x1BEZNiyQ== +"@babel/plugin-transform-block-scoped-functions@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.14.5.tgz#e48641d999d4bc157a67ef336aeb54bc44fd3ad4" + integrity sha512-dtqWqdWZ5NqBX3KzsVCWfQI3A53Ft5pWFCT2eCVUftWZgjc5DpDponbIF1+c+7cSGk2wN0YK7HGL/ezfRbpKBQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-classes@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.12.1.tgz#65e650fcaddd3d88ddce67c0f834a3d436a32db6" - integrity sha512-/74xkA7bVdzQTBeSUhLLJgYIcxw/dpEpCdRDiHgPJ3Mv6uC11UhjpOhl72CgqbBCmt1qtssCyB2xnJm1+PFjog== +"@babel/plugin-transform-block-scoping@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.14.5.tgz#8cc63e61e50f42e078e6f09be775a75f23ef9939" + integrity sha512-LBYm4ZocNgoCqyxMLoOnwpsmQ18HWTQvql64t3GvMUzLQrNoV1BDG0lNftC8QKYERkZgCCT/7J5xWGObGAyHDw== dependencies: - "@babel/helper-annotate-as-pure" "^7.10.4" - "@babel/helper-define-map" "^7.10.4" - "@babel/helper-function-name" "^7.10.4" - "@babel/helper-optimise-call-expression" "^7.10.4" - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-replace-supers" "^7.12.1" - "@babel/helper-split-export-declaration" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" + +"@babel/plugin-transform-classes@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.14.5.tgz#0e98e82097b38550b03b483f9b51a78de0acb2cf" + integrity sha512-J4VxKAMykM06K/64z9rwiL6xnBHgB1+FVspqvlgCdwD1KUbQNfszeKVVOMh59w3sztHYIZDgnhOC4WbdEfHFDA== + dependencies: + "@babel/helper-annotate-as-pure" "^7.14.5" + "@babel/helper-function-name" "^7.14.5" + "@babel/helper-optimise-call-expression" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-replace-supers" "^7.14.5" + "@babel/helper-split-export-declaration" "^7.14.5" globals "^11.1.0" -"@babel/plugin-transform-computed-properties@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.12.1.tgz#d68cf6c9b7f838a8a4144badbe97541ea0904852" - integrity sha512-vVUOYpPWB7BkgUWPo4C44mUQHpTZXakEqFjbv8rQMg7TC6S6ZhGZ3otQcRH6u7+adSlE5i0sp63eMC/XGffrzg== +"@babel/plugin-transform-computed-properties@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.14.5.tgz#1b9d78987420d11223d41195461cc43b974b204f" + integrity sha512-pWM+E4283UxaVzLb8UBXv4EIxMovU4zxT1OPnpHJcmnvyY9QbPPTKZfEj31EUvG3/EQRbYAGaYEUZ4yWOBC2xg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-destructuring@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.12.1.tgz#b9a570fe0d0a8d460116413cb4f97e8e08b2f847" - integrity sha512-fRMYFKuzi/rSiYb2uRLiUENJOKq4Gnl+6qOv5f8z0TZXg3llUwUhsNNwrwaT/6dUhJTzNpBr+CUvEWBtfNY1cw== +"@babel/plugin-transform-destructuring@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.14.5.tgz#d32ad19ff1a6da1e861dc62720d80d9776e3bf35" + integrity sha512-wU9tYisEbRMxqDezKUqC9GleLycCRoUsai9ddlsq54r8QRLaeEhc+d+9DqCG+kV9W2GgQjTZESPTpn5bAFMDww== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-dotall-regex@^7.12.1", "@babel/plugin-transform-dotall-regex@^7.4.4": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.12.1.tgz#a1d16c14862817b6409c0a678d6f9373ca9cd975" - integrity sha512-B2pXeRKoLszfEW7J4Hg9LoFaWEbr/kzo3teWHmtFCszjRNa/b40f9mfeqZsIDLLt/FjwQ6pz/Gdlwy85xNckBA== +"@babel/plugin-transform-dotall-regex@^7.14.5", "@babel/plugin-transform-dotall-regex@^7.4.4": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.14.5.tgz#2f6bf76e46bdf8043b4e7e16cf24532629ba0c7a" + integrity sha512-loGlnBdj02MDsFaHhAIJzh7euK89lBrGIdM9EAtHFo6xKygCUGuuWe07o1oZVk287amtW1n0808sQM99aZt3gw== dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-create-regexp-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-duplicate-keys@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.12.1.tgz#745661baba295ac06e686822797a69fbaa2ca228" - integrity sha512-iRght0T0HztAb/CazveUpUQrZY+aGKKaWXMJ4uf9YJtqxSUe09j3wteztCUDRHs+SRAL7yMuFqUsLoAKKzgXjw== +"@babel/plugin-transform-duplicate-keys@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.14.5.tgz#365a4844881bdf1501e3a9f0270e7f0f91177954" + integrity sha512-iJjbI53huKbPDAsJ8EmVmvCKeeq21bAze4fu9GBQtSLqfvzj2oRuHVx4ZkDwEhg1htQ+5OBZh/Ab0XDf5iBZ7A== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-exponentiation-operator@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.12.1.tgz#b0f2ed356ba1be1428ecaf128ff8a24f02830ae0" - integrity sha512-7tqwy2bv48q+c1EHbXK0Zx3KXd2RVQp6OC7PbwFNt/dPTAV3Lu5sWtWuAj8owr5wqtWnqHfl2/mJlUmqkChKug== +"@babel/plugin-transform-exponentiation-operator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.14.5.tgz#5154b8dd6a3dfe6d90923d61724bd3deeb90b493" + integrity sha512-jFazJhMBc9D27o9jDnIE5ZErI0R0m7PbKXVq77FFvqFbzvTMuv8jaAwLZ5PviOLSFttqKIW0/wxNSDbjLk0tYA== dependencies: - "@babel/helper-builder-binary-assignment-operator-visitor" "^7.10.4" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-builder-binary-assignment-operator-visitor" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-for-of@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.12.1.tgz#07640f28867ed16f9511c99c888291f560921cfa" - integrity sha512-Zaeq10naAsuHo7heQvyV0ptj4dlZJwZgNAtBYBnu5nNKJoW62m0zKcIEyVECrUKErkUkg6ajMy4ZfnVZciSBhg== +"@babel/plugin-transform-for-of@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.14.5.tgz#dae384613de8f77c196a8869cbf602a44f7fc0eb" + integrity sha512-CfmqxSUZzBl0rSjpoQSFoR9UEj3HzbGuGNL21/iFTmjb5gFggJp3ph0xR1YBhexmLoKRHzgxuFvty2xdSt6gTA== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-function-name@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.12.1.tgz#2ec76258c70fe08c6d7da154003a480620eba667" - integrity sha512-JF3UgJUILoFrFMEnOJLJkRHSk6LUSXLmEFsA23aR2O5CSLUxbeUX1IZ1YQ7Sn0aXb601Ncwjx73a+FVqgcljVw== +"@babel/plugin-transform-function-name@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.14.5.tgz#e81c65ecb900746d7f31802f6bed1f52d915d6f2" + integrity sha512-vbO6kv0fIzZ1GpmGQuvbwwm+O4Cbm2NrPzwlup9+/3fdkuzo1YqOZcXw26+YUJB84Ja7j9yURWposEHLYwxUfQ== dependencies: - "@babel/helper-function-name" "^7.10.4" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-function-name" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-literals@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.12.1.tgz#d73b803a26b37017ddf9d3bb8f4dc58bfb806f57" - integrity sha512-+PxVGA+2Ag6uGgL0A5f+9rklOnnMccwEBzwYFL3EUaKuiyVnUipyXncFcfjSkbimLrODoqki1U9XxZzTvfN7IQ== +"@babel/plugin-transform-literals@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.14.5.tgz#41d06c7ff5d4d09e3cf4587bd3ecf3930c730f78" + integrity sha512-ql33+epql2F49bi8aHXxvLURHkxJbSmMKl9J5yHqg4PLtdE6Uc48CH1GS6TQvZ86eoB/ApZXwm7jlA+B3kra7A== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-member-expression-literals@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.12.1.tgz#496038602daf1514a64d43d8e17cbb2755e0c3ad" - integrity sha512-1sxePl6z9ad0gFMB9KqmYofk34flq62aqMt9NqliS/7hPEpURUCMbyHXrMPlo282iY7nAvUB1aQd5mg79UD9Jg== +"@babel/plugin-transform-member-expression-literals@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.14.5.tgz#b39cd5212a2bf235a617d320ec2b48bcc091b8a7" + integrity sha512-WkNXxH1VXVTKarWFqmso83xl+2V3Eo28YY5utIkbsmXoItO8Q3aZxN4BTS2k0hz9dGUloHK26mJMyQEYfkn/+Q== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-modules-amd@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.12.1.tgz#3154300b026185666eebb0c0ed7f8415fefcf6f9" - integrity sha512-tDW8hMkzad5oDtzsB70HIQQRBiTKrhfgwC/KkJeGsaNFTdWhKNt/BiE8c5yj19XiGyrxpbkOfH87qkNg1YGlOQ== +"@babel/plugin-transform-modules-amd@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.14.5.tgz#4fd9ce7e3411cb8b83848480b7041d83004858f7" + integrity sha512-3lpOU8Vxmp3roC4vzFpSdEpGUWSMsHFreTWOMMLzel2gNGfHE5UWIh/LN6ghHs2xurUp4jRFYMUIZhuFbody1g== dependencies: - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-module-transforms" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-commonjs@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.12.1.tgz#fa403124542636c786cf9b460a0ffbb48a86e648" - integrity sha512-dY789wq6l0uLY8py9c1B48V8mVL5gZh/+PQ5ZPrylPYsnAvnEMjqsUXkuoDVPeVK+0VyGar+D08107LzDQ6pag== +"@babel/plugin-transform-modules-commonjs@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.14.5.tgz#7aaee0ea98283de94da98b28f8c35701429dad97" + integrity sha512-en8GfBtgnydoao2PS+87mKyw62k02k7kJ9ltbKe0fXTHrQmG6QZZflYuGI1VVG7sVpx4E1n7KBpNlPb8m78J+A== dependencies: - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-simple-access" "^7.12.1" + "@babel/helper-module-transforms" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-simple-access" "^7.14.5" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-systemjs@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.12.1.tgz#663fea620d593c93f214a464cd399bf6dc683086" - integrity sha512-Hn7cVvOavVh8yvW6fLwveFqSnd7rbQN3zJvoPNyNaQSvgfKmDBO9U1YL9+PCXGRlZD9tNdWTy5ACKqMuzyn32Q== +"@babel/plugin-transform-modules-systemjs@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.14.5.tgz#c75342ef8b30dcde4295d3401aae24e65638ed29" + integrity sha512-mNMQdvBEE5DcMQaL5LbzXFMANrQjd2W7FPzg34Y4yEz7dBgdaC+9B84dSO+/1Wba98zoDbInctCDo4JGxz1VYA== dependencies: - "@babel/helper-hoist-variables" "^7.10.4" - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-validator-identifier" "^7.10.4" + "@babel/helper-hoist-variables" "^7.14.5" + "@babel/helper-module-transforms" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-validator-identifier" "^7.14.5" babel-plugin-dynamic-import-node "^2.3.3" -"@babel/plugin-transform-modules-umd@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.12.1.tgz#eb5a218d6b1c68f3d6217b8fa2cc82fec6547902" - integrity sha512-aEIubCS0KHKM0zUos5fIoQm+AZUMt1ZvMpqz0/H5qAQ7vWylr9+PLYurT+Ic7ID/bKLd4q8hDovaG3Zch2uz5Q== +"@babel/plugin-transform-modules-umd@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.14.5.tgz#fb662dfee697cce274a7cda525190a79096aa6e0" + integrity sha512-RfPGoagSngC06LsGUYyM9QWSXZ8MysEjDJTAea1lqRjNECE3y0qIJF/qbvJxc4oA4s99HumIMdXOrd+TdKaAAA== dependencies: - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-module-transforms" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-named-capturing-groups-regex@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.1.tgz#b407f5c96be0d9f5f88467497fa82b30ac3e8753" - integrity sha512-tB43uQ62RHcoDp9v2Nsf+dSM8sbNodbEicbQNA53zHz8pWUhsgHSJCGpt7daXxRydjb0KnfmB+ChXOv3oADp1Q== +"@babel/plugin-transform-named-capturing-groups-regex@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.14.5.tgz#d537e8ee083ee6f6aa4f4eef9d2081d555746e4c" + integrity sha512-+Xe5+6MWFo311U8SchgeX5c1+lJM+eZDBZgD+tvXu9VVQPXwwVzeManMMjYX6xw2HczngfOSZjoFYKwdeB/Jvw== dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.12.1" + "@babel/helper-create-regexp-features-plugin" "^7.14.5" -"@babel/plugin-transform-new-target@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.1.tgz#80073f02ee1bb2d365c3416490e085c95759dec0" - integrity sha512-+eW/VLcUL5L9IvJH7rT1sT0CzkdUTvPrXC2PXTn/7z7tXLBuKvezYbGdxD5WMRoyvyaujOq2fWoKl869heKjhw== +"@babel/plugin-transform-new-target@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.14.5.tgz#31bdae8b925dc84076ebfcd2a9940143aed7dbf8" + integrity sha512-Nx054zovz6IIRWEB49RDRuXGI4Gy0GMgqG0cII9L3MxqgXz/+rgII+RU58qpo4g7tNEx1jG7rRVH4ihZoP4esQ== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-object-super@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.12.1.tgz#4ea08696b8d2e65841d0c7706482b048bed1066e" - integrity sha512-AvypiGJH9hsquNUn+RXVcBdeE3KHPZexWRdimhuV59cSoOt5kFBmqlByorAeUlGG2CJWd0U+4ZtNKga/TB0cAw== +"@babel/plugin-transform-object-super@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.14.5.tgz#d0b5faeac9e98597a161a9cf78c527ed934cdc45" + integrity sha512-MKfOBWzK0pZIrav9z/hkRqIk/2bTv9qvxHzPQc12RcVkMOzpIKnFCNYJip00ssKWYkd8Sf5g0Wr7pqJ+cmtuFg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-replace-supers" "^7.12.1" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-replace-supers" "^7.14.5" -"@babel/plugin-transform-parameters@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.12.1.tgz#d2e963b038771650c922eff593799c96d853255d" - integrity sha512-xq9C5EQhdPK23ZeCdMxl8bbRnAgHFrw5EOC3KJUsSylZqdkCaFEXxGSBuTSObOpiiHHNyb82es8M1QYgfQGfNg== +"@babel/plugin-transform-parameters@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.14.5.tgz#49662e86a1f3ddccac6363a7dfb1ff0a158afeb3" + integrity sha512-Tl7LWdr6HUxTmzQtzuU14SqbgrSKmaR77M0OKyq4njZLQTPfOvzblNKyNkGwOfEFCEx7KeYHQHDI0P3F02IVkA== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-property-literals@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.12.1.tgz#41bc81200d730abb4456ab8b3fbd5537b59adecd" - integrity sha512-6MTCR/mZ1MQS+AwZLplX4cEySjCpnIF26ToWo942nqn8hXSm7McaHQNeGx/pt7suI1TWOWMfa/NgBhiqSnX0cQ== +"@babel/plugin-transform-property-literals@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.14.5.tgz#0ddbaa1f83db3606f1cdf4846fa1dfb473458b34" + integrity sha512-r1uilDthkgXW8Z1vJz2dKYLV1tuw2xsbrp3MrZmD99Wh9vsfKoob+JTgri5VUb/JqyKRXotlOtwgu4stIYCmnw== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-regenerator@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.12.1.tgz#5f0a28d842f6462281f06a964e88ba8d7ab49753" - integrity sha512-gYrHqs5itw6i4PflFX3OdBPMQdPbF4bj2REIUxlMRUFk0/ZOAIpDFuViuxPjUL7YC8UPnf+XG7/utJvqXdPKng== +"@babel/plugin-transform-regenerator@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.14.5.tgz#9676fd5707ed28f522727c5b3c0aa8544440b04f" + integrity sha512-NVIY1W3ITDP5xQl50NgTKlZ0GrotKtLna08/uGY6ErQt6VEQZXla86x/CTddm5gZdcr+5GSsvMeTmWA5Ii6pkg== dependencies: regenerator-transform "^0.14.2" -"@babel/plugin-transform-reserved-words@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.12.1.tgz#6fdfc8cc7edcc42b36a7c12188c6787c873adcd8" - integrity sha512-pOnUfhyPKvZpVyBHhSBoX8vfA09b7r00Pmm1sH+29ae2hMTKVmSp4Ztsr8KBKjLjx17H0eJqaRC3bR2iThM54A== +"@babel/plugin-transform-reserved-words@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.14.5.tgz#c44589b661cfdbef8d4300dcc7469dffa92f8304" + integrity sha512-cv4F2rv1nD4qdexOGsRQXJrOcyb5CrgjUH9PKrrtyhSDBNWGxd0UIitjyJiWagS+EbUGjG++22mGH1Pub8D6Vg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-shorthand-properties@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.12.1.tgz#0bf9cac5550fce0cfdf043420f661d645fdc75e3" - integrity sha512-GFZS3c/MhX1OusqB1MZ1ct2xRzX5ppQh2JU1h2Pnfk88HtFTM+TWQqJNfwkmxtPQtb/s1tk87oENfXJlx7rSDw== +"@babel/plugin-transform-shorthand-properties@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.14.5.tgz#97f13855f1409338d8cadcbaca670ad79e091a58" + integrity sha512-xLucks6T1VmGsTB+GWK5Pl9Jl5+nRXD1uoFdA5TSO6xtiNjtXTjKkmPdFXVLGlK5A2/or/wQMKfmQ2Y0XJfn5g== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-spread@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.12.1.tgz#527f9f311be4ec7fdc2b79bb89f7bf884b3e1e1e" - integrity sha512-vuLp8CP0BE18zVYjsEBZ5xoCecMK6LBMMxYzJnh01rxQRvhNhH1csMMmBfNo5tGpGO+NhdSNW2mzIvBu3K1fng== +"@babel/plugin-transform-spread@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.14.5.tgz#bd269fb4119754d2ce7f4cc39a96b4f71baae356" + integrity sha512-/3iqoQdiWergnShZYl0xACb4ADeYCJ7X/RgmwtXshn6cIvautRPAFzhd58frQlokLO6Jb4/3JXvmm6WNTPtiTw== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-skip-transparent-expression-wrappers" "^7.14.5" -"@babel/plugin-transform-sticky-regex@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.7.tgz#560224613ab23987453948ed21d0b0b193fa7fad" - integrity sha512-VEiqZL5N/QvDbdjfYQBhruN0HYjSPjC4XkeqW4ny/jNtH9gcbgaqBIXYEZCNnESMAGs0/K/R7oFGMhOyu/eIxg== +"@babel/plugin-transform-sticky-regex@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.14.5.tgz#5b617542675e8b7761294381f3c28c633f40aeb9" + integrity sha512-Z7F7GyvEMzIIbwnziAZmnSNpdijdr4dWt+FJNBnBLz5mwDFkqIXU9wmBcWWad3QeJF5hMTkRe4dAq2sUZiG+8A== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-template-literals@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.12.1.tgz#b43ece6ed9a79c0c71119f576d299ef09d942843" - integrity sha512-b4Zx3KHi+taXB1dVRBhVJtEPi9h1THCeKmae2qP0YdUHIFhVjtpqqNfxeVAa1xeHVhAy4SbHxEwx5cltAu5apw== +"@babel/plugin-transform-template-literals@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.14.5.tgz#a5f2bc233937d8453885dc736bdd8d9ffabf3d93" + integrity sha512-22btZeURqiepOfuy/VkFr+zStqlujWaarpMErvay7goJS6BWwdd6BY9zQyDLDa4x2S3VugxFb162IZ4m/S/+Gg== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-typeof-symbol@^7.12.10": - version "7.12.10" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.10.tgz#de01c4c8f96580bd00f183072b0d0ecdcf0dec4b" - integrity sha512-JQ6H8Rnsogh//ijxspCjc21YPd3VLVoYtAwv3zQmqAt8YGYUtdo5usNhdl4b9/Vir2kPFZl6n1h0PfUz4hJhaA== +"@babel/plugin-transform-typeof-symbol@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.14.5.tgz#39af2739e989a2bd291bf6b53f16981423d457d4" + integrity sha512-lXzLD30ffCWseTbMQzrvDWqljvZlHkXU+CnseMhkMNqU1sASnCsz3tSzAaH3vCUXb9PHeUb90ZT1BdFTm1xxJw== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-unicode-escapes@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.12.1.tgz#5232b9f81ccb07070b7c3c36c67a1b78f1845709" - integrity sha512-I8gNHJLIc7GdApm7wkVnStWssPNbSRMPtgHdmH3sRM1zopz09UWPS4x5V4n1yz/MIWTVnJ9sp6IkuXdWM4w+2Q== +"@babel/plugin-transform-unicode-escapes@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.14.5.tgz#9d4bd2a681e3c5d7acf4f57fa9e51175d91d0c6b" + integrity sha512-crTo4jATEOjxj7bt9lbYXcBAM3LZaUrbP2uUdxb6WIorLmjNKSpHfIybgY4B8SRpbf8tEVIWH3Vtm7ayCrKocA== dependencies: - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-plugin-utils" "^7.14.5" -"@babel/plugin-transform-unicode-regex@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.12.1.tgz#cc9661f61390db5c65e3febaccefd5c6ac3faecb" - integrity sha512-SqH4ClNngh/zGwHZOOQMTD+e8FGWexILV+ePMyiDJttAWRh5dhDL8rcl5lSgU3Huiq6Zn6pWTMvdPAb21Dwdyg== +"@babel/plugin-transform-unicode-regex@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.14.5.tgz#4cd09b6c8425dd81255c7ceb3fb1836e7414382e" + integrity sha512-UygduJpC5kHeCiRw/xDVzC+wj8VaYSoKl5JNVmbP7MadpNinAm3SvZCxZ42H37KZBKztz46YC73i9yV34d0Tzw== dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.12.1" - "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-create-regexp-features-plugin" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" "@babel/preset-env@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.12.11.tgz#55d5f7981487365c93dbbc84507b1c7215e857f9" - integrity sha512-j8Tb+KKIXKYlDBQyIOy4BLxzv1NUOwlHfZ74rvW+Z0Gp4/cI2IMDPBWAgWceGcE7aep9oL/0K9mlzlMGxA8yNw== + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.14.5.tgz#c0c84e763661fd0e74292c3d511cb33b0c668997" + integrity sha512-ci6TsS0bjrdPpWGnQ+m4f+JSSzDKlckqKIJJt9UZ/+g7Zz9k0N8lYU8IeLg/01o2h8LyNZDMLGgRLDTxpudLsA== dependencies: - "@babel/compat-data" "^7.12.7" - "@babel/helper-compilation-targets" "^7.12.5" - "@babel/helper-module-imports" "^7.12.5" - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-validator-option" "^7.12.11" - "@babel/plugin-proposal-async-generator-functions" "^7.12.1" - "@babel/plugin-proposal-class-properties" "^7.12.1" - "@babel/plugin-proposal-dynamic-import" "^7.12.1" - "@babel/plugin-proposal-export-namespace-from" "^7.12.1" - "@babel/plugin-proposal-json-strings" "^7.12.1" - "@babel/plugin-proposal-logical-assignment-operators" "^7.12.1" - "@babel/plugin-proposal-nullish-coalescing-operator" "^7.12.1" - "@babel/plugin-proposal-numeric-separator" "^7.12.7" - "@babel/plugin-proposal-object-rest-spread" "^7.12.1" - "@babel/plugin-proposal-optional-catch-binding" "^7.12.1" - "@babel/plugin-proposal-optional-chaining" "^7.12.7" - "@babel/plugin-proposal-private-methods" "^7.12.1" - "@babel/plugin-proposal-unicode-property-regex" "^7.12.1" - "@babel/plugin-syntax-async-generators" "^7.8.0" - "@babel/plugin-syntax-class-properties" "^7.12.1" - "@babel/plugin-syntax-dynamic-import" "^7.8.0" + "@babel/compat-data" "^7.14.5" + "@babel/helper-compilation-targets" "^7.14.5" + "@babel/helper-plugin-utils" "^7.14.5" + "@babel/helper-validator-option" "^7.14.5" + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining" "^7.14.5" + "@babel/plugin-proposal-async-generator-functions" "^7.14.5" + "@babel/plugin-proposal-class-properties" "^7.14.5" + "@babel/plugin-proposal-class-static-block" "^7.14.5" + "@babel/plugin-proposal-dynamic-import" "^7.14.5" + "@babel/plugin-proposal-export-namespace-from" "^7.14.5" + "@babel/plugin-proposal-json-strings" "^7.14.5" + "@babel/plugin-proposal-logical-assignment-operators" "^7.14.5" + "@babel/plugin-proposal-nullish-coalescing-operator" "^7.14.5" + "@babel/plugin-proposal-numeric-separator" "^7.14.5" + "@babel/plugin-proposal-object-rest-spread" "^7.14.5" + "@babel/plugin-proposal-optional-catch-binding" "^7.14.5" + "@babel/plugin-proposal-optional-chaining" "^7.14.5" + "@babel/plugin-proposal-private-methods" "^7.14.5" + "@babel/plugin-proposal-private-property-in-object" "^7.14.5" + "@babel/plugin-proposal-unicode-property-regex" "^7.14.5" + "@babel/plugin-syntax-async-generators" "^7.8.4" + "@babel/plugin-syntax-class-properties" "^7.12.13" + "@babel/plugin-syntax-class-static-block" "^7.14.5" + "@babel/plugin-syntax-dynamic-import" "^7.8.3" "@babel/plugin-syntax-export-namespace-from" "^7.8.3" - "@babel/plugin-syntax-json-strings" "^7.8.0" + "@babel/plugin-syntax-json-strings" "^7.8.3" "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" - "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0" + "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" "@babel/plugin-syntax-numeric-separator" "^7.10.4" - "@babel/plugin-syntax-object-rest-spread" "^7.8.0" - "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" - "@babel/plugin-syntax-optional-chaining" "^7.8.0" - "@babel/plugin-syntax-top-level-await" "^7.12.1" - "@babel/plugin-transform-arrow-functions" "^7.12.1" - "@babel/plugin-transform-async-to-generator" "^7.12.1" - "@babel/plugin-transform-block-scoped-functions" "^7.12.1" - "@babel/plugin-transform-block-scoping" "^7.12.11" - "@babel/plugin-transform-classes" "^7.12.1" - "@babel/plugin-transform-computed-properties" "^7.12.1" - "@babel/plugin-transform-destructuring" "^7.12.1" - "@babel/plugin-transform-dotall-regex" "^7.12.1" - "@babel/plugin-transform-duplicate-keys" "^7.12.1" - "@babel/plugin-transform-exponentiation-operator" "^7.12.1" - "@babel/plugin-transform-for-of" "^7.12.1" - "@babel/plugin-transform-function-name" "^7.12.1" - "@babel/plugin-transform-literals" "^7.12.1" - "@babel/plugin-transform-member-expression-literals" "^7.12.1" - "@babel/plugin-transform-modules-amd" "^7.12.1" - "@babel/plugin-transform-modules-commonjs" "^7.12.1" - "@babel/plugin-transform-modules-systemjs" "^7.12.1" - "@babel/plugin-transform-modules-umd" "^7.12.1" - "@babel/plugin-transform-named-capturing-groups-regex" "^7.12.1" - "@babel/plugin-transform-new-target" "^7.12.1" - "@babel/plugin-transform-object-super" "^7.12.1" - "@babel/plugin-transform-parameters" "^7.12.1" - "@babel/plugin-transform-property-literals" "^7.12.1" - "@babel/plugin-transform-regenerator" "^7.12.1" - "@babel/plugin-transform-reserved-words" "^7.12.1" - "@babel/plugin-transform-shorthand-properties" "^7.12.1" - "@babel/plugin-transform-spread" "^7.12.1" - "@babel/plugin-transform-sticky-regex" "^7.12.7" - "@babel/plugin-transform-template-literals" "^7.12.1" - "@babel/plugin-transform-typeof-symbol" "^7.12.10" - "@babel/plugin-transform-unicode-escapes" "^7.12.1" - "@babel/plugin-transform-unicode-regex" "^7.12.1" - "@babel/preset-modules" "^0.1.3" - "@babel/types" "^7.12.11" - core-js-compat "^3.8.0" - semver "^5.5.0" + "@babel/plugin-syntax-object-rest-spread" "^7.8.3" + "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" + "@babel/plugin-syntax-optional-chaining" "^7.8.3" + "@babel/plugin-syntax-private-property-in-object" "^7.14.5" + "@babel/plugin-syntax-top-level-await" "^7.14.5" + "@babel/plugin-transform-arrow-functions" "^7.14.5" + "@babel/plugin-transform-async-to-generator" "^7.14.5" + "@babel/plugin-transform-block-scoped-functions" "^7.14.5" + "@babel/plugin-transform-block-scoping" "^7.14.5" + "@babel/plugin-transform-classes" "^7.14.5" + "@babel/plugin-transform-computed-properties" "^7.14.5" + "@babel/plugin-transform-destructuring" "^7.14.5" + "@babel/plugin-transform-dotall-regex" "^7.14.5" + "@babel/plugin-transform-duplicate-keys" "^7.14.5" + "@babel/plugin-transform-exponentiation-operator" "^7.14.5" + "@babel/plugin-transform-for-of" "^7.14.5" + "@babel/plugin-transform-function-name" "^7.14.5" + "@babel/plugin-transform-literals" "^7.14.5" + "@babel/plugin-transform-member-expression-literals" "^7.14.5" + "@babel/plugin-transform-modules-amd" "^7.14.5" + "@babel/plugin-transform-modules-commonjs" "^7.14.5" + "@babel/plugin-transform-modules-systemjs" "^7.14.5" + "@babel/plugin-transform-modules-umd" "^7.14.5" + "@babel/plugin-transform-named-capturing-groups-regex" "^7.14.5" + "@babel/plugin-transform-new-target" "^7.14.5" + "@babel/plugin-transform-object-super" "^7.14.5" + "@babel/plugin-transform-parameters" "^7.14.5" + "@babel/plugin-transform-property-literals" "^7.14.5" + "@babel/plugin-transform-regenerator" "^7.14.5" + "@babel/plugin-transform-reserved-words" "^7.14.5" + "@babel/plugin-transform-shorthand-properties" "^7.14.5" + "@babel/plugin-transform-spread" "^7.14.5" + "@babel/plugin-transform-sticky-regex" "^7.14.5" + "@babel/plugin-transform-template-literals" "^7.14.5" + "@babel/plugin-transform-typeof-symbol" "^7.14.5" + "@babel/plugin-transform-unicode-escapes" "^7.14.5" + "@babel/plugin-transform-unicode-regex" "^7.14.5" + "@babel/preset-modules" "^0.1.4" + "@babel/types" "^7.14.5" + babel-plugin-polyfill-corejs2 "^0.2.2" + babel-plugin-polyfill-corejs3 "^0.2.2" + babel-plugin-polyfill-regenerator "^0.2.2" + core-js-compat "^3.14.0" + semver "^6.3.0" -"@babel/preset-modules@^0.1.3": +"@babel/preset-modules@^0.1.4": version "0.1.4" resolved "https://registry.yarnpkg.com/@babel/preset-modules/-/preset-modules-0.1.4.tgz#362f2b68c662842970fdb5e254ffc8fc1c2e415e" integrity sha512-J36NhwnfdzpmH41M1DrnkkgAqhZaqr/NBdPfQ677mLzlaXo+oDiv1deyCDtgAhz8p328otdob0Du7+xgHGZbKg== @@ -780,52 +830,42 @@ esutils "^2.0.2" "@babel/runtime@^7.8.4": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.12.5.tgz#410e7e487441e1b360c29be715d870d9b985882e" - integrity sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg== + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.5.tgz#665450911c6031af38f81db530f387ec04cd9a98" + integrity sha512-121rumjddw9c3NCQ55KGkyE1h/nzWhU/owjhw0l4mQrkzz4x9SGS1X8gFLraHwX7td3Yo4QTL+qj0NcIzN87BA== dependencies: regenerator-runtime "^0.13.4" -"@babel/template@^7.10.4", "@babel/template@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.7.tgz#c817233696018e39fbb6c491d2fb684e05ed43bc" - integrity sha512-GkDzmHS6GV7ZeXfJZ0tLRBhZcMcY0/Lnb+eEbXDBfCAcZCjrZKe6p3J4we/D24O9Y8enxWAg1cWwof59yLh2ow== +"@babel/template@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.14.5.tgz#a9bc9d8b33354ff6e55a9c60d1109200a68974f4" + integrity sha512-6Z3Po85sfxRGachLULUhOmvAaOo7xCvqGQtxINai2mEGPFm6pQ4z5QInFnUrRpfoSV60BnjyF5F3c+15fxFV1g== dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/parser" "^7.12.7" - "@babel/types" "^7.12.7" + "@babel/code-frame" "^7.14.5" + "@babel/parser" "^7.14.5" + "@babel/types" "^7.14.5" -"@babel/traverse@^7.10.4", "@babel/traverse@^7.12.1", "@babel/traverse@^7.12.10", "@babel/traverse@^7.12.5": - version "7.12.12" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.12.tgz#d0cd87892704edd8da002d674bc811ce64743376" - integrity sha512-s88i0X0lPy45RrLM8b9mz8RPH5FqO9G9p7ti59cToE44xFm1Q+Pjh5Gq4SXBbtb88X7Uy7pexeqRIQDDMNkL0w== +"@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.5.tgz#c111b0f58afab4fea3d3385a406f692748c59870" + integrity sha512-G3BiS15vevepdmFqmUc9X+64y0viZYygubAMO8SvBmKARuF6CPSZtH4Ng9vi/lrWlZFGe3FWdXNy835akH8Glg== dependencies: - "@babel/code-frame" "^7.12.11" - "@babel/generator" "^7.12.11" - "@babel/helper-function-name" "^7.12.11" - "@babel/helper-split-export-declaration" "^7.12.11" - "@babel/parser" "^7.12.11" - "@babel/types" "^7.12.12" + "@babel/code-frame" "^7.14.5" + "@babel/generator" "^7.14.5" + "@babel/helper-function-name" "^7.14.5" + "@babel/helper-hoist-variables" "^7.14.5" + "@babel/helper-split-export-declaration" "^7.14.5" + "@babel/parser" "^7.14.5" + "@babel/types" "^7.14.5" debug "^4.1.0" globals "^11.1.0" - lodash "^4.17.19" -"@babel/types@^7.10.4", "@babel/types@^7.10.5", "@babel/types@^7.12.1", "@babel/types@^7.12.10", "@babel/types@^7.12.11", "@babel/types@^7.12.12", "@babel/types@^7.12.5", "@babel/types@^7.12.7", "@babel/types@^7.4.4": - version "7.12.12" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.12.tgz#4608a6ec313abbd87afa55004d373ad04a96c299" - integrity sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ== +"@babel/types@^7.14.5", "@babel/types@^7.4.4", "@babel/types@^7.9.6": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.14.5.tgz#3bb997ba829a2104cedb20689c4a5b8121d383ff" + integrity sha512-M/NzBpEL95I5Hh4dwhin5JlE7EzO5PHMAuzjxss3tiOBD46KfQvVedN/3jEPZvdRvtsK2222XfdHogNIttFgcg== dependencies: - "@babel/helper-validator-identifier" "^7.12.11" - lodash "^4.17.19" - to-fast-properties "^2.0.0" - -"@babel/types@^7.9.6": - version "7.13.0" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.0.tgz#74424d2816f0171b4100f0ab34e9a374efdf7f80" - integrity sha512-hE+HE8rnG1Z6Wzo+MhaKE5lM5eMx71T4EHJgku2E3xIfaULhDcxiiRxUYgwX8qwP1BBSlag+TdGOt6JAidIZTA== - dependencies: - "@babel/helper-validator-identifier" "^7.12.11" - lodash "^4.17.19" + "@babel/helper-validator-identifier" "^7.14.5" to-fast-properties "^2.0.0" "@balena/dockerignore@^1.0.2": @@ -834,14 +874,14 @@ integrity sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q== "@discoveryjs/json-ext@^0.5.0": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.2.tgz#8f03a22a04de437254e8ce8cc84ba39689288752" - integrity sha512-HyYEUDeIj5rRQU2Hk5HTB2uHsbRQpF70nvMhVzi+VJR0X+xNEhjPui4/kBf3VeH/wqD28PT4sVOm8qqLjBrSZg== + version "0.5.3" + resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.3.tgz#90420f9f9c6d3987f176a19a7d8e764271a2f55d" + integrity sha512-Fxt+AfXgjMoin2maPIYzFZnQjAXjAL0PHscM5pRTtatFqB+vZxAM9tLp2Optnuw3QOQC40jTNeGYFOMvyf7v9g== "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6": - version "7.0.6" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" - integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== + version "7.0.7" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" + integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== "@webassemblyjs/ast@1.9.0": version "1.9.0" @@ -988,17 +1028,22 @@ "@webassemblyjs/wast-parser" "1.9.0" "@xtuc/long" "4.2.2" -"@webpack-cli/info@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@webpack-cli/info/-/info-1.2.0.tgz#6051d6adf3618df664f4945a2b76355c00f83f0d" - integrity sha512-+wA8lBKopgKmN76BSGJVJby5ZXDlsrO6p/nm7fUBsHznRNWB/ozotJP7Yfcz8JPfqeG2LxwYlTH2u6D9a/0XAw== +"@webpack-cli/configtest@^1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@webpack-cli/configtest/-/configtest-1.0.4.tgz#f03ce6311c0883a83d04569e2c03c6238316d2aa" + integrity sha512-cs3XLy+UcxiP6bj0A6u7MLLuwdXJ1c3Dtc0RkKg+wiI1g/Ti1om8+/2hc2A2B60NbBNAbMgyBMHvyymWm/j4wQ== + +"@webpack-cli/info@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@webpack-cli/info/-/info-1.3.0.tgz#9d78a31101a960997a4acd41ffd9b9300627fe2b" + integrity sha512-ASiVB3t9LOKHs5DyVUcxpraBXDOKubYu/ihHhU+t1UPpxsivg6Od2E2qU4gJCekfEddzRBzHhzA/Acyw/mlK/w== dependencies: envinfo "^7.7.3" -"@webpack-cli/serve@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-1.2.0.tgz#8cb2c1e95426f5caed1f3bf9d7ccf3ea41d85f52" - integrity sha512-jI3P7jMp/AXDSPkM+ClwRcJZbxnlvNC8bVZBmyRr4scMMZ4p5WQcXkw3Q+Hc7RQekomJlBMN+UQGliT4hhG8Vw== +"@webpack-cli/serve@^1.5.1": + version "1.5.1" + resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-1.5.1.tgz#b5fde2f0f79c1e120307c415a4c1d5eb15a6f278" + integrity sha512-4vSVUiOPJLmr45S8rMGy7WDvpWxfFxfP/Qx/cxZFCfvoypTYpPPL1X8VIZMe0WTA+Jr7blUxwUSEZNkjoMTgSw== "@xtuc/ieee754@^1.2.0": version "1.2.0" @@ -1043,11 +1088,6 @@ ajv@^6.1.0, ajv@^6.10.2, ajv@^6.12.4, ajv@^6.12.5: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ansi-colors@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" - integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== - ansi-regex@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" @@ -1069,9 +1109,9 @@ anymatch@^2.0.0: normalize-path "^2.1.1" anymatch@~3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.1.tgz#c55ecf02185e2469259399310c173ce31233b142" - integrity sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg== + version "3.1.2" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" + integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== dependencies: normalize-path "^3.0.0" picomatch "^2.0.4" @@ -1140,9 +1180,9 @@ async-limiter@~1.0.0: integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== async-lock@^1.2.6: - version "1.2.6" - resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.2.6.tgz#c83c7a2569d1745306f4a5ae03680310e5f65e67" - integrity sha512-gobUp/bRWL/uJsxi4ZK7NM770s5d2Tx5Hl7uxFIcN6yTz1Kvy2RCSKEvzhLsjAAnYaNa8lDvcjy9ybM6lXFjIg== + version "1.3.0" + resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.3.0.tgz#0fba111bea8b9693020857eba4f9adca173df3e5" + integrity sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg== async@0.9.x: version "0.9.2" @@ -1171,6 +1211,30 @@ babel-plugin-dynamic-import-node@^2.3.3: dependencies: object.assign "^4.1.0" +babel-plugin-polyfill-corejs2@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.2.2.tgz#e9124785e6fd94f94b618a7954e5693053bf5327" + integrity sha512-kISrENsJ0z5dNPq5eRvcctITNHYXWOA4DUZRFYCz3jYCcvTb/A546LIddmoGNMVYg2U38OyFeNosQwI9ENTqIQ== + dependencies: + "@babel/compat-data" "^7.13.11" + "@babel/helper-define-polyfill-provider" "^0.2.2" + semver "^6.1.1" + +babel-plugin-polyfill-corejs3@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.2.2.tgz#7424a1682ee44baec817327710b1b094e5f8f7f5" + integrity sha512-l1Cf8PKk12eEk5QP/NQ6TH8A1pee6wWDJ96WjxrMXFLHLOBFzYM4moG80HFgduVhTqAFez4alnZKEhP/bYHg0A== + dependencies: + "@babel/helper-define-polyfill-provider" "^0.2.2" + core-js-compat "^3.9.1" + +babel-plugin-polyfill-regenerator@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.2.2.tgz#b310c8d642acada348c1fa3b3e6ce0e851bee077" + integrity sha512-Goy5ghsc21HgPDFtzRkSirpZVW35meGoTmTOb2bxqdl60ghub4xOidgNTHaZfQ2FaxQsKmwvXtOAkcIS4SMBWg== + dependencies: + "@babel/helper-define-polyfill-provider" "^0.2.2" + babel-walk@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/babel-walk/-/babel-walk-3.0.0.tgz#71ea81ea1b8e2e7b37540ec8c899fe49d7fb1441" @@ -1179,9 +1243,9 @@ babel-walk@^3.0.0: "@babel/types" "^7.9.6" balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== base64-js@^1.0.2, base64-js@^1.3.1: version "1.5.1" @@ -1212,9 +1276,9 @@ binary-extensions@^1.0.0: integrity sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw== binary-extensions@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" - integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== + version "2.2.0" + resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" + integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== bindings@^1.5.0: version "1.5.0" @@ -1228,15 +1292,15 @@ bluebird@^3.5.5: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.4.0: - version "4.11.9" - resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.9.tgz#26d556829458f9d1e81fc48952493d0ba3507828" - integrity sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw== +bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9: + version "4.12.0" + resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88" + integrity sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA== bn.js@^5.0.0, bn.js@^5.1.1: - version "5.1.3" - resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.3.tgz#beca005408f642ebebea80b042b4d18d2ac0ee6b" - integrity sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ== + version "5.2.0" + resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.0.tgz#358860674396c6997771a9d051fcc1b57d4ae002" + integrity sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw== body-parser@1.19.0: version "1.19.0" @@ -1285,7 +1349,7 @@ braces@~3.0.2: dependencies: fill-range "^7.0.1" -brorand@^1.0.1: +brorand@^1.0.1, brorand@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f" integrity sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8= @@ -1351,16 +1415,16 @@ browserify-zlib@^0.2.0: dependencies: pako "~1.0.5" -browserslist@^4.14.5, browserslist@^4.15.0: - version "4.16.0" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.0.tgz#410277627500be3cb28a1bfe037586fbedf9488b" - integrity sha512-/j6k8R0p3nxOC6kx5JGAxsnhc9ixaWJfYc+TNTzxg6+ARaESAvQGV7h0uNOB4t+pLQJZWzcrMxXOxjgsCj3dqQ== +browserslist@^4.16.6: + version "4.16.6" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.6.tgz#d7901277a5a88e554ed305b183ec9b0c08f66fa2" + integrity sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ== dependencies: - caniuse-lite "^1.0.30001165" - colorette "^1.2.1" - electron-to-chromium "^1.3.621" + caniuse-lite "^1.0.30001219" + colorette "^1.2.2" + electron-to-chromium "^1.3.723" escalade "^3.1.1" - node-releases "^1.1.67" + node-releases "^1.1.71" buffer-from@^1.0.0: version "1.1.1" @@ -1436,22 +1500,17 @@ cache-base@^1.0.1: unset-value "^1.0.0" call-bind@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.0.tgz#24127054bb3f9bdcb4b1fb82418186072f77b8ce" - integrity sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w== + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== dependencies: function-bind "^1.1.1" - get-intrinsic "^1.0.0" + get-intrinsic "^1.0.2" -camelcase@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809" - integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== - -caniuse-lite@^1.0.30001165: - version "1.0.30001170" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001170.tgz#0088bfecc6a14694969e391cc29d7eb6362ca6a7" - integrity sha512-Dd4d/+0tsK0UNLrZs3CvNukqalnVTRrxb5mcQm8rHL49t7V5ZaTygwXkrq+FB+dVDf++4ri8eJnFEJAB8332PA== +caniuse-lite@^1.0.30001219: + version "1.0.30001237" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz#4b7783661515b8e7151fc6376cfd97f0e427b9e5" + integrity sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw== chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" @@ -1482,9 +1541,9 @@ chokidar@^2.1.8: fsevents "^1.2.7" chokidar@^3.4.1: - version "3.4.3" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.3.tgz#c1df38231448e45ca4ac588e6c79573ba6a57d5b" - integrity sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ== + version "3.5.1" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a" + integrity sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw== dependencies: anymatch "~3.1.1" braces "~3.0.2" @@ -1494,7 +1553,7 @@ chokidar@^3.4.1: normalize-path "~3.0.0" readdirp "~3.5.0" optionalDependencies: - fsevents "~2.1.2" + fsevents "~2.3.1" chownr@^1.1.1: version "1.1.4" @@ -1502,11 +1561,9 @@ chownr@^1.1.1: integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== chrome-trace-event@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz#234090ee97c7d4ad1a2c4beae27505deffc608a4" - integrity sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ== - dependencies: - tslib "^1.9.0" + version "1.0.3" + resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" + integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" @@ -1526,6 +1583,15 @@ class-utils@^0.3.5: isobject "^3.0.0" static-extend "^0.1.1" +clone-deep@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" + integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ== + dependencies: + is-plain-object "^2.0.4" + kind-of "^6.0.2" + shallow-clone "^3.0.0" + collection-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -1546,25 +1612,20 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= -colorette@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" - integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== +colorette@^1.2.1, colorette@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" + integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== -commander@^6.2.0: - version "6.2.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" - integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== - -commander@^7.1.0: - version "7.1.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.1.0.tgz#f2eaecf131f10e36e07d894698226e36ae0eb5ff" - integrity sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg== +commander@^7.0.0, commander@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== commondir@^1.0.1: version "1.0.1" @@ -1647,12 +1708,12 @@ copy-descriptor@^0.1.0: resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= -core-js-compat@^3.8.0: - version "3.8.1" - resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.8.1.tgz#8d1ddd341d660ba6194cbe0ce60f4c794c87a36e" - integrity sha512-a16TLmy9NVD1rkjUGbwuyWkiDoN0FDpAwrfLONvHFQx0D9k7J9y0srwMT8QP/Z6HE3MIFaVynEeYwZwPX1o5RQ== +core-js-compat@^3.14.0, core-js-compat@^3.9.1: + version "3.14.0" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.14.0.tgz#b574dabf29184681d5b16357bd33d104df3d29a5" + integrity sha512-R4NS2eupxtiJU+VwgkF9WTpnSfZW4pogwKHd8bclWU2sp93Pr5S1uYJI84cMOubJRou7bcfL0vmwtLslWN5p3A== dependencies: - browserslist "^4.15.0" + browserslist "^4.16.6" semver "7.0.0" core-util-is@~1.0.0: @@ -1691,7 +1752,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" -cross-spawn@^7.0.0: +cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== @@ -1718,22 +1779,20 @@ crypto-browserify@^3.11.0: randomfill "^1.0.3" css-loader@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-5.0.1.tgz#9e4de0d6636a6266a585bd0900b422c85539d25f" - integrity sha512-cXc2ti9V234cq7rJzFKhirb2L2iPy8ZjALeVJAozXYz9te3r4eqLSixNAbMDJSgJEQywqXzs8gonxaboeKqwiw== + version "5.2.6" + resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-5.2.6.tgz#c3c82ab77fea1f360e587d871a6811f4450cc8d1" + integrity sha512-0wyN5vXMQZu6BvjbrPdUJvkCzGEO24HC7IS7nW4llc6BBFC+zwR9CKtYGv63Puzsg10L/o12inMY5/2ByzfD6w== dependencies: - camelcase "^6.2.0" - cssesc "^3.0.0" - icss-utils "^5.0.0" + icss-utils "^5.1.0" loader-utils "^2.0.0" - postcss "^8.1.4" + postcss "^8.2.15" postcss-modules-extract-imports "^3.0.0" postcss-modules-local-by-default "^4.0.0" postcss-modules-scope "^3.0.0" postcss-modules-values "^4.0.0" postcss-value-parser "^4.1.0" schema-utils "^3.0.0" - semver "^7.3.2" + semver "^7.3.5" cssesc@^3.0.0: version "3.0.0" @@ -1746,9 +1805,9 @@ cyclist@^1.0.1: integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= debounce@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.0.tgz#44a540abc0ea9943018dc0eaa95cce87f65cd131" - integrity sha512-mYtLl1xfZLi1m4RtQYlZgJUNQjl4ZxVnHzIR8nLLgi4q1YT8o/WM+MK/f8yfcc9s5Ir5zRaPZyZU6xs1Syoocg== + version "1.2.1" + resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.1.tgz#38881d8f4166a5c5848020c11827b834bcb3e0a5" + integrity sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug== debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" @@ -1757,7 +1816,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3: dependencies: ms "2.0.0" -debug@^4.1.0: +debug@^4.1.0, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== @@ -1851,29 +1910,29 @@ ee-first@1.1.1: integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= ejs@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.5.tgz#aed723844dc20acb4b170cd9ab1017e476a0d93b" - integrity sha512-dldq3ZfFtgVTJMLjOe+/3sROTzALlL9E34V4/sDtUd/KlBSS0s6U1/+WPE1B4sj9CXHJpL1M6rhNJnc9Wbal9w== + version "3.1.6" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.6.tgz#5bfd0a0689743bb5268b3550cceeebbc1702822a" + integrity sha512-9lt9Zse4hPucPkoP7FHDF0LQAlGyF9JVpnClFLFH3aSSbxmyoqINRpp/9wePWJTUl4KOQwRL72Iw3InHPDkoGw== dependencies: jake "^10.6.1" -electron-to-chromium@^1.3.621: - version "1.3.633" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.633.tgz#16dd5aec9de03894e8d14a1db4cda8a369b9b7fe" - integrity sha512-bsVCsONiVX1abkWdH7KtpuDAhsQ3N3bjPYhROSAXE78roJKet0Y5wznA14JE9pzbwSZmSMAW6KiKYf1RvbTJkA== +electron-to-chromium@^1.3.723: + version "1.3.752" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.752.tgz#0728587f1b9b970ec9ffad932496429aef750d09" + integrity sha512-2Tg+7jSl3oPxgsBsWKh5H83QazTkmWG/cnNwJplmyZc7KcN61+I10oUgaXSVk/NwfvN3BdkKDR4FYuRBQQ2v0A== elliptic@^6.5.3: - version "6.5.3" - resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.3.tgz#cb59eb2efdaf73a0bd78ccd7015a62ad6e0f93d6" - integrity sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw== + version "6.5.4" + resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb" + integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ== dependencies: - bn.js "^4.4.0" - brorand "^1.0.1" + bn.js "^4.11.9" + brorand "^1.1.0" hash.js "^1.0.0" - hmac-drbg "^1.0.0" - inherits "^2.0.1" - minimalistic-assert "^1.0.0" - minimalistic-crypto-utils "^1.0.0" + hmac-drbg "^1.0.1" + inherits "^2.0.4" + minimalistic-assert "^1.0.1" + minimalistic-crypto-utils "^1.0.1" emojis-list@^3.0.0: version "3.0.0" @@ -1892,26 +1951,19 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" -enhanced-resolve@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz#3b806f3bfafc1ec7de69551ef93cca46c1704126" - integrity sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ== +enhanced-resolve@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz#2f3cfd84dbe3b487f18f2db2ef1e064a571ca5ec" + integrity sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg== dependencies: graceful-fs "^4.1.2" memory-fs "^0.5.0" tapable "^1.0.0" -enquirer@^2.3.6: - version "2.3.6" - resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" - integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== - dependencies: - ansi-colors "^4.1.1" - envinfo@^7.7.3: - version "7.7.3" - resolved "https://registry.yarnpkg.com/envinfo/-/envinfo-7.7.3.tgz#4b2d8622e3e7366afb8091b23ed95569ea0208cc" - integrity sha512-46+j5QxbPWza0PB1i15nZx0xQ4I/EfQxg9J8Had3b408SV63nEtor2e+oiY63amTo9KTuh2a3XLObNwduxYwwA== + version "7.8.1" + resolved "https://registry.yarnpkg.com/envinfo/-/envinfo-7.8.1.tgz#06377e3e5f4d379fea7ac592d5ad8927e0c4d475" + integrity sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw== errno@^0.1.3, errno@~0.1.7: version "0.1.8" @@ -1976,9 +2028,9 @@ eventemitter3@^4.0.4: integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== events@^3.0.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/events/-/events-3.2.0.tgz#93b87c18f8efcd4202a461aec4dfc0556b639379" - integrity sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg== + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: version "1.0.3" @@ -1988,19 +2040,19 @@ evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: md5.js "^1.3.4" safe-buffer "^5.1.1" -execa@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" - integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== +execa@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" + integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== dependencies: - cross-spawn "^7.0.0" - get-stream "^5.0.0" - human-signals "^1.1.1" + cross-spawn "^7.0.3" + get-stream "^6.0.0" + human-signals "^2.1.0" is-stream "^2.0.0" merge-stream "^2.0.0" - npm-run-path "^4.0.0" - onetime "^5.1.0" - signal-exit "^3.0.2" + npm-run-path "^4.0.1" + onetime "^5.1.2" + signal-exit "^3.0.3" strip-final-newline "^2.0.0" expand-brackets@^2.1.4: @@ -2122,9 +2174,9 @@ file-uri-to-path@1.0.0: integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== filelist@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.1.tgz#f10d1a3ae86c1694808e8f20906f43d4c9132dbb" - integrity sha512-8zSK6Nu0DQIC08mUC46sWGXi+q3GGpKydAG36k+JDba6VRpkevvOWUW5a/PhShij4+vHT9M+ghgG7eM+a9JDUQ== + version "1.0.2" + resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.2.tgz#80202f21462d4d1c2e214119b1807c1bc0380e5b" + integrity sha512-z7O0IS8Plc39rTCq6i6iHxk43duYOn8uFJiWSewIq0Bww1RNybVHSCjahmcC87ZqAm4OTvFzlzeGu3XAzG1ctQ== dependencies: minimatch "^3.0.4" @@ -2204,10 +2256,10 @@ for-in@^1.0.2: resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= -forwarded@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" - integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ= +forwarded@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== fragment-cache@^0.2.1: version "0.2.1" @@ -2262,36 +2314,34 @@ fsevents@^1.2.7: bindings "^1.5.0" nan "^2.12.1" -fsevents@~2.1.2: - version "2.1.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" - integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fsevents@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== -gensync@^1.0.0-beta.1: +gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== -get-intrinsic@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.0.2.tgz#6820da226e50b24894e08859469dc68361545d49" - integrity sha512-aeX0vrFm21ILl3+JpFFRNe9aUvp6VFZb2/CTbgLb8j75kOhvoNYjt9d8KA/tJG4gSo8nzEDedRl0h7vDmBYRVg== +get-intrinsic@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" + integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== dependencies: function-bind "^1.1.1" has "^1.0.3" has-symbols "^1.0.1" -get-stream@^5.0.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== - dependencies: - pump "^3.0.0" +get-stream@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" + integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" @@ -2307,9 +2357,9 @@ glob-parent@^3.1.0: path-dirname "^1.0.0" glob-parent@~5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229" - integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ== + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== dependencies: is-glob "^4.0.1" @@ -2319,9 +2369,9 @@ glob-to-regexp@^0.3.0: integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= glob@^7.1.3, glob@^7.1.4: - version "7.1.6" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" - integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== + version "7.1.7" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" + integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -2336,9 +2386,9 @@ globals@^11.1.0: integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6: - version "4.2.4" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" - integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== + version "4.2.6" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" + integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== has-flag@^3.0.0: version "3.0.0" @@ -2346,9 +2396,9 @@ has-flag@^3.0.0: integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= has-symbols@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.1.tgz#9f5214758a44196c406d9bd76cebf81ec2dd31e8" - integrity sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg== + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423" + integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw== has-value@^0.3.1: version "0.3.1" @@ -2412,7 +2462,7 @@ historic-readline@^1.0.8: dependencies: fs-extra "^0.24.0" -hmac-drbg@^1.0.0: +hmac-drbg@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1" integrity sha1-0nRXAQJabHdabFRXk+1QL8DGSaE= @@ -2448,10 +2498,10 @@ https-browserify@^1.0.0: resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM= -human-signals@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" - integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== +human-signals@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" + integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== iconv-lite@0.4.24: version "0.4.24" @@ -2460,7 +2510,7 @@ iconv-lite@0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -icss-utils@^5.0.0: +icss-utils@^5.0.0, icss-utils@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== @@ -2488,11 +2538,6 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -indexes-of@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607" - integrity sha1-8w9xbI4r00bHtn0985FVZqfAVgc= - infer-owner@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" @@ -2564,10 +2609,10 @@ is-buffer@^1.1.5: resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== -is-core-module@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a" - integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ== +is-core-module@^2.2.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.4.0.tgz#8e9fc8e15027b011418026e98f0e6f4d86305cc1" + integrity sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A== dependencies: has "^1.0.3" @@ -2733,9 +2778,9 @@ json5@^1.0.1: minimist "^1.2.0" json5@^2.1.2: - version "2.1.3" - resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.3.tgz#c9b0f7fa9233bfe5807fe66fcf3a5617ed597d43" - integrity sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA== + version "2.2.0" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" + integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== dependencies: minimist "^1.2.5" @@ -2813,10 +2858,15 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" -lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: - version "4.17.20" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" - integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== +lodash.debounce@^4.0.8: + version "4.0.8" + resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" + integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= + +lodash@^4.17.20: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== lru-cache@^5.1.1: version "5.1.1" @@ -2931,17 +2981,17 @@ miller-rabin@^4.0.0: bn.js "^4.0.0" brorand "^1.0.1" -mime-db@1.44.0: - version "1.44.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" - integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +mime-db@1.48.0: + version "1.48.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" + integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== mime-types@~2.1.24: - version "2.1.27" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" - integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== + version "2.1.31" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" + integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg== dependencies: - mime-db "1.44.0" + mime-db "1.48.0" mime@1.6.0: version "1.6.0" @@ -2958,7 +3008,7 @@ minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1: resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz#2e194de044626d4a10e7f7fbc00ce73e83e4d5c7" integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== -minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1: +minimalistic-crypto-utils@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" integrity sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo= @@ -3060,10 +3110,10 @@ nan@^2.12.1, nan@^2.14.0: resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== -nanoid@^3.1.20: - version "3.1.20" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.20.tgz#badc263c6b1dcf14b71efaa85f6ab4c1d6cfc788" - integrity sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw== +nanoid@^3.1.23: + version "3.1.23" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.23.tgz#f744086ce7c2bc47ee0a8472574d5c78e4183a81" + integrity sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw== nanomatch@^1.2.9: version "1.2.13" @@ -3128,10 +3178,10 @@ node-pty@^0.9.0: dependencies: nan "^2.14.0" -node-releases@^1.1.67: - version "1.1.67" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.67.tgz#28ebfcccd0baa6aad8e8d4d8fe4cbc49ae239c12" - integrity sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg== +node-releases@^1.1.71: + version "1.1.73" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.73.tgz#dd4e81ddd5277ff846b80b52bb40c49edf7a7b20" + integrity sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg== normalize-path@^2.1.1: version "2.1.1" @@ -3145,7 +3195,7 @@ normalize-path@^3.0.0, normalize-path@~3.0.0: resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== -npm-run-path@^4.0.0: +npm-run-path@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== @@ -3209,7 +3259,7 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" -onetime@^5.1.0: +onetime@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== @@ -3338,9 +3388,9 @@ path-key@^3.0.0, path-key@^3.1.0: integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== path-parse@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" - integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== path-to-regexp@0.1.7: version "0.1.7" @@ -3348,9 +3398,9 @@ path-to-regexp@0.1.7: integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= pbkdf2@^3.0.3: - version "3.1.1" - resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.1.1.tgz#cb8724b0fada984596856d1a6ebafd3584654b94" - integrity sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg== + version "3.1.2" + resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.1.2.tgz#dd822aa0887580e52f1a039dc3eda108efae3075" + integrity sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA== dependencies: create-hash "^1.1.2" create-hmac "^1.1.4" @@ -3359,9 +3409,9 @@ pbkdf2@^3.0.3: sha.js "^2.4.8" picomatch@^2.0.4, picomatch@^2.2.1: - version "2.2.2" - resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" - integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg== + version "2.3.0" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" + integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== pify@^4.0.1: version "4.0.1" @@ -3416,13 +3466,11 @@ postcss-modules-values@^4.0.0: icss-utils "^5.0.0" postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4: - version "6.0.4" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz#56075a1380a04604c38b063ea7767a129af5c2b3" - integrity sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw== + version "6.0.6" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.6.tgz#2c5bba8174ac2f6981ab631a42ab0ee54af332ea" + integrity sha512-9LXrvaaX3+mcv5xkg5kFwqSzSH1JIObIx51PrndZwlmznwXRfxMddDvo9gve3gVR8ZTKgoFDdWkbRFmEhT4PMg== dependencies: cssesc "^3.0.0" - indexes-of "^1.0.1" - uniq "^1.0.1" util-deprecate "^1.0.2" postcss-value-parser@^4.1.0: @@ -3430,14 +3478,14 @@ postcss-value-parser@^4.1.0: resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz#443f6a20ced6481a2bda4fa8532a6e55d789a2cb" integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ== -postcss@^8.1.4: - version "8.2.1" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.1.tgz#eabc5557c4558059b9d9e5b15bce7ffa9089c2a8" - integrity sha512-RhsqOOAQzTgh1UB/IZdca7F9WDb7SUCR2Vnv1x7DbvuuggQIpoDwjK+q0rzoPffhYvWNKX5JSwS4so4K3UC6vA== +postcss@^8.2.15: + version "8.3.2" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.2.tgz#ed3ec489f5428af5740cd6effcc216b4d455ee64" + integrity sha512-y1FK/AWdZlBF5lusS5j5l4/vF67+vQZt1SXPVJ32y1kRGDQyrs1zk32hG1cInRTu14P0V+orPz+ifwW/7rR4bg== dependencies: - colorette "^1.2.1" - nanoid "^3.1.20" - source-map "^0.6.1" + colorette "^1.2.2" + nanoid "^3.1.23" + source-map-js "^0.6.2" process-nextick-args@~2.0.0: version "2.0.1" @@ -3455,11 +3503,11 @@ promise-inflight@^1.0.1: integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM= proxy-addr@~2.0.5: - version "2.0.6" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf" - integrity sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw== + version "2.0.7" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== dependencies: - forwarded "~0.1.2" + forwarded "0.2.0" ipaddr.js "1.9.1" prr@~1.0.1: @@ -3659,9 +3707,9 @@ regjsgen@^0.5.1: integrity sha512-OFFT3MfrH90xIW8OOSyUrk6QHD5E9JOTeGodiJeBS3J6IwlgzJMNE/1bZklWz5oTg+9dCMyEetclvCVXOPoN3A== regjsparser@^0.6.4: - version "0.6.4" - resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.4.tgz#a769f8684308401a66e9b529d2436ff4d0666272" - integrity sha512-64O87/dPDgfk8/RQqC4gkZoGyyWFIEUTTh80CU6CWuK5vkCGyekIx+oKcEIYtP/RAxSQltCZHCNu/mdd7fqlJw== + version "0.6.9" + resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.9.tgz#b489eef7c9a2ce43727627011429cf833a7183e6" + integrity sha512-ZqbNRz1SNjLAiYuwY0zoXW8Ne675IX5q+YHioAGbCw4X96Mjl2+dcX9B2ciaeyYjViDAfvIjFpQjJgLttTEERQ== dependencies: jsesc "~0.5.0" @@ -3671,9 +3719,9 @@ remove-trailing-separator@^1.0.1: integrity sha1-wkvOKig62tW8P1jg1IJJuSN52O8= repeat-element@^1.1.2: - version "1.1.3" - resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.3.tgz#782e0d825c0c5a3bb39731f84efee6b742e6b1ce" - integrity sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g== + version "1.1.4" + resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.4.tgz#be681520847ab58c7568ac75fbfad28ed42d39e9" + integrity sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ== repeat-string@^1.6.1: version "1.6.1" @@ -3697,12 +3745,12 @@ resolve-url@^0.2.1: resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= -resolve@^1.9.0: - version "1.19.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" - integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg== +resolve@^1.14.2, resolve@^1.9.0: + version "1.20.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" + integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== dependencies: - is-core-module "^2.1.0" + is-core-module "^2.2.0" path-parse "^1.0.6" ret@~0.1.10: @@ -3786,20 +3834,20 @@ semver@7.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== -semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: +semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== -semver@^6.0.0, semver@^6.3.0: +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.3.2: - version "7.3.4" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.4.tgz#27aaa7d2e4ca76452f98d3add093a72c943edc97" - integrity sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw== +semver@^7.3.5: + version "7.3.5" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" + integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== dependencies: lru-cache "^6.0.0" @@ -3867,6 +3915,13 @@ sha.js@^2.4.0, sha.js@^2.4.8: inherits "^2.0.1" safe-buffer "^5.0.1" +shallow-clone@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" + integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA== + dependencies: + kind-of "^6.0.2" + shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" @@ -3884,7 +3939,7 @@ shell-quote@^1.7.2: resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2" integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== -signal-exit@^3.0.2: +signal-exit@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== @@ -3924,6 +3979,11 @@ source-list-map@^2.0.0: resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== +source-map-js@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-0.6.2.tgz#0bb5de631b41cfbda6cfba8bd05a80efdfd2385e" + integrity sha512-/3GptzWzu0+0MBQFrDKzw/DvvMTUORvgY6k6jd/VS6iCR4RDTKWH6v6WPwQoUO8667uQEf9Oe38DxAYWY5F/Ug== + source-map-resolve@^0.5.0: version "0.5.3" resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a" @@ -3944,9 +4004,9 @@ source-map-support@~0.5.12: source-map "^0.6.0" source-map-url@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" - integrity sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM= + version "0.4.1" + resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.1.tgz#0af66605a745a5a2f91cf1bbf8a7afbc283dec56" + integrity sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw== source-map@^0.5.0, source-map@^0.5.6: version "0.5.7" @@ -3966,9 +4026,9 @@ split-string@^3.0.1, split-string@^3.0.2: extend-shallow "^3.0.0" ssri@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.1.tgz#2a3c41b28dd45b62b63676ecb74001265ae9edd8" - integrity sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA== + version "6.0.2" + resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.2.tgz#157939134f20464e7301ddba3e90ffa8f7728ac5" + integrity sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q== dependencies: figgy-pudding "^3.5.1" @@ -4149,11 +4209,6 @@ toidentifier@1.0.0: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== -tslib@^1.9.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" - integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== - tty-browserify@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6" @@ -4205,11 +4260,6 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" -uniq@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff" - integrity sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8= - unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -4243,9 +4293,9 @@ upath@^1.1.1: integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== uri-js@^4.2.2: - version "4.4.0" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.0.tgz#aa714261de793e8a82347a7bcc9ce74e86f28602" - integrity sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g== + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" @@ -4297,9 +4347,9 @@ uuid@^8.3.2: integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== v8-compile-cache@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132" - integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q== + version "2.3.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" + integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== vary@~1.1.2: version "1.1.2" @@ -4311,23 +4361,18 @@ vm-browserify@^1.0.1: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ== -vscode-jsonrpc@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz#108bdb09b4400705176b957ceca9e0880e9b6d4e" - integrity sha512-wnJA4BnEjOSyFMvjZdpiOwhSq9uDoK8e/kpRJDTaMYzwlkrhG1fwDIZI94CLsLzlCK5cIbMMtFlJlfR57Lavmg== - vscode-jsonrpc@^5.0.0, vscode-jsonrpc@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz#9bab9c330d89f43fc8c1e8702b5c36e058a01794" integrity sha512-JvONPptw3GAQGXlVV2utDcHx0BiY34FupW/kI6mZ5x06ER5DdPG/tXWMVHjTNULF5uKPOUUD0SaXg5QaubJL0A== vscode-languageclient@^6.0.0: - version "6.1.3" - resolved "https://registry.yarnpkg.com/vscode-languageclient/-/vscode-languageclient-6.1.3.tgz#c979c5bb5855714a0307e998c18ca827c1b3953a" - integrity sha512-YciJxk08iU5LmWu7j5dUt9/1OLjokKET6rME3cI4BRpiF6HZlusm2ZwPt0MYJ0lV5y43sZsQHhyon2xBg4ZJVA== + version "6.1.4" + resolved "https://registry.yarnpkg.com/vscode-languageclient/-/vscode-languageclient-6.1.4.tgz#54aa8b1559ae2e0499cb6ab746cc2662fb6ecc0f" + integrity sha512-EUOU+bJu6axmt0RFNo3nrglQLPXMfanbYViJee3Fbn2VuQoX0ZOI4uTYhSRvYLP2vfwTP/juV62P/mksCdTZMA== dependencies: semver "^6.3.0" - vscode-languageserver-protocol "^3.15.3" + vscode-languageserver-protocol "3.15.3" vscode-languageserver-protocol@3.15.3: version "3.15.3" @@ -4337,24 +4382,11 @@ vscode-languageserver-protocol@3.15.3: vscode-jsonrpc "^5.0.1" vscode-languageserver-types "3.15.1" -vscode-languageserver-protocol@^3.15.3: - version "3.16.0" - resolved "https://registry.yarnpkg.com/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0.tgz#34135b61a9091db972188a07d337406a3cdbe821" - integrity sha512-sdeUoAawceQdgIfTI+sdcwkiK2KU+2cbEYA0agzM2uqaUy2UpnnGHtWTHVEtS0ES4zHU0eMFRGN+oQgDxlD66A== - dependencies: - vscode-jsonrpc "6.0.0" - vscode-languageserver-types "3.16.0" - vscode-languageserver-types@3.15.1: version "3.15.1" resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz#17be71d78d2f6236d414f0001ce1ef4d23e6b6de" integrity sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ== -vscode-languageserver-types@3.16.0: - version "3.16.0" - resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz#ecf393fc121ec6974b2da3efb3155644c514e247" - integrity sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA== - vscode-uri@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-2.1.2.tgz#c8d40de93eb57af31f3c715dd650e2ca2c096f1c" @@ -4379,30 +4411,31 @@ watchpack@^1.7.4: watchpack-chokidar2 "^2.0.1" webpack-cli@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.3.0.tgz#e39303bf9f8002de122903e97029f3443d0f9174" - integrity sha512-gve+BBKrzMPTOYDjupzV8JchUznhVWMKtWM1hFIQWi6XoeLvGNoQwkrtMWVb+aJ437GgCKdta7sIn10v621pKA== + version "4.7.2" + resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.7.2.tgz#a718db600de6d3906a4357e059ae584a89f4c1a5" + integrity sha512-mEoLmnmOIZQNiRl0ebnjzQ74Hk0iKS5SiEEnpq3dRezoyR3yPaeQZCMCe+db4524pj1Pd5ghZXjT41KLzIhSLw== dependencies: "@discoveryjs/json-ext" "^0.5.0" - "@webpack-cli/info" "^1.2.0" - "@webpack-cli/serve" "^1.2.0" + "@webpack-cli/configtest" "^1.0.4" + "@webpack-cli/info" "^1.3.0" + "@webpack-cli/serve" "^1.5.1" colorette "^1.2.1" - commander "^6.2.0" - enquirer "^2.3.6" - execa "^4.1.0" + commander "^7.0.0" + execa "^5.0.0" fastest-levenshtein "^1.0.12" import-local "^3.0.2" interpret "^2.2.0" rechoir "^0.7.0" v8-compile-cache "^2.2.0" - webpack-merge "^4.2.2" + webpack-merge "^5.7.3" -webpack-merge@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-4.2.2.tgz#a27c52ea783d1398afd2087f547d7b9d2f43634d" - integrity sha512-TUE1UGoTX2Cd42j3krGYqObZbOD+xF7u28WB7tfUordytSjbWTIjK/8V0amkBfTYN4/pB/GIDlJZZ657BGG19g== +webpack-merge@^5.7.3: + version "5.8.0" + resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-5.8.0.tgz#2b39dbf22af87776ad744c390223731d30a68f61" + integrity sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q== dependencies: - lodash "^4.17.15" + clone-deep "^4.0.1" + wildcard "^2.0.0" webpack-sources@^1.4.0, webpack-sources@^1.4.1: version "1.4.3" @@ -4413,9 +4446,9 @@ webpack-sources@^1.4.0, webpack-sources@^1.4.1: source-map "~0.6.1" webpack@^4.44.2: - version "4.44.2" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.44.2.tgz#6bfe2b0af055c8b2d1e90ed2cd9363f841266b72" - integrity sha512-6KJVGlCxYdISyurpQ0IPTklv+DULv05rs2hseIXer6D7KrUicRDLFb4IUM1S6LUAKypPM/nSiVSuv8jHu1m3/Q== + version "4.46.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.46.0.tgz#bf9b4404ea20a073605e0a011d188d77cb6ad542" + integrity sha512-6jJuJjg8znb/xRItk7bkT0+Q7AHCYjjFnvKIWQPkNIOyRqoCGvkOs0ipeQzrqz4l5FtN5ZI/ukEHroeX/o1/5Q== dependencies: "@webassemblyjs/ast" "1.9.0" "@webassemblyjs/helper-module-context" "1.9.0" @@ -4425,7 +4458,7 @@ webpack@^4.44.2: ajv "^6.10.2" ajv-keywords "^3.4.1" chrome-trace-event "^1.0.2" - enhanced-resolve "^4.3.0" + enhanced-resolve "^4.5.0" eslint-scope "^4.0.3" json-parse-better-errors "^1.0.2" loader-runner "^2.4.0" @@ -4448,6 +4481,11 @@ which@^2.0.1: dependencies: isexe "^2.0.0" +wildcard@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.0.tgz#a77d20e5200c6faaac979e4b3aadc7b3dd7f8fec" + integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== + worker-farm@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8" @@ -4461,9 +4499,9 @@ wrappy@1: integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= ws@^5.2.0: - version "5.2.2" - resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.2.tgz#dffef14866b8e8dc9133582514d1befaf96e980f" - integrity sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA== + version "5.2.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.3.tgz#05541053414921bc29c63bee14b8b0dd50b07b3d" + integrity sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA== dependencies: async-limiter "~1.0.0" @@ -4478,14 +4516,14 @@ xterm-addon-fit@^0.4.0: integrity sha512-p4BESuV/g2L6pZzFHpeNLLnep9mp/DkF3qrPglMiucSFtD8iJxtMufEoEJbN8LZwB4i+8PFpFvVuFrGOSpW05w== xterm@^4.9.0: - version "4.9.0" - resolved "https://registry.yarnpkg.com/xterm/-/xterm-4.9.0.tgz#7a4c097a433d565339b5533b468bbc60c6c87969" - integrity sha512-wGfqufmioctKr8VkbRuZbVDfjlXWGZZ1PWHy1yqqpGT3Nm6yaJx8lxDbSEBANtgaiVPTcKSp97sxOy5IlpqYfw== + version "4.13.0" + resolved "https://registry.yarnpkg.com/xterm/-/xterm-4.13.0.tgz#7998de1e2ad92c4796fe45807be4f31061f3d9d1" + integrity sha512-HVW1gdoLOTnkMaqQCr2r3mQy4fX9iSa5gWxKZ2UTYdLa4iqavv7QxJ8n1Ypse32shPVkhTYPLS6vHEFZp5ghzw== y18n@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.1.tgz#8db2b83c31c5d75099bb890b23f3094891e247d4" - integrity sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ== + version "4.0.3" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf" + integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ== yallist@^3.0.2: version "3.1.1" @@ -4498,6 +4536,6 @@ yallist@^4.0.0: integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== yaml@^1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.0.tgz#3b593add944876077d4d683fee01081bd9fff31e" - integrity sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg== + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== From a7bc26f875cd8d70377186d26f0a8d0f8f31983a Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 21:15:15 +0000 Subject: [PATCH 044/104] Update README --- CONTRIBUTING.md | 11 +++++++++++ README.md | 44 ++++++++++++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..443b218 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,11 @@ +# Contributing guide + +* [Criteria for language inclusion](doc/what-languages.md) +* [How to add your own language to Riju](doc/tutorial.md) +* [Deep dive on Riju build system](doc/build.md) +* [Deploying your own instance of Riju](doc/infrastructure.md) + +If you'd like to request a new language, head to the [language support +meta-issue](https://github.com/raxod502/riju/issues/24) and add a +comment. Of course, if you actually want it to be added anytime soon, +you should submit a pull request :) diff --git a/README.md b/README.md index 4b30c81..8c33997 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,37 @@ Riju is a very fast online playground for every programming language. In less than a second, you can start playing with a Python interpreter -or compiling INTERCAL code. +or compiling [INTERCAL](https://en.wikipedia.org/wiki/INTERCAL) code. -Check out the [live application](https://riju.codes/)! +Check it out at the ! -**You should not write any sensitive code on Riju, as NO GUARANTEES -are made about the security or privacy of your data. (No warranty etc -etc.)** +## Is it free? -This project is a work in progress, and I don't intend on thoroughly -documenting it until it has reached feature-completeness. +Riju is free and always will be free for everyone. -## Documentation +However, if Riju gets popular enough, I won't be able to afford paying +for the hosting myself. To help me keep Riju online, you can donate +via Patreon. All donations are used solely to cover hosting costs, and +any surplus is donated to the [Electronic Frontier +Foundation](https://www.eff.org/). -* [Criteria for language inclusion](doc/what-languages.md) -* [How to add your own language to Riju](doc/tutorial.md) -* [Deep dive on Riju build system](doc/build.md) -* [Deploying your own instance of Riju](doc/infrastructure.md) +## Is it safe? -If you'd like to request a new language, head to the [language support -meta-issue](https://github.com/raxod502/riju/issues/24) and add a -comment. Of course, if you actually want it to be added anytime soon, -you should submit a pull request :) +Riju does not collect your personal information. + +* Your code is deleted from the server as soon as you close Riju. +* Your terminal input and output is never saved or logged anywhere. +* Riju uses [Fathom Analytics](https://usefathom.com/) to measure + traffic. Fathom collects very limited data and does not sell it to + third parties, unlike Google Analytics. +* Riju does not serve advertisements. + +All of the above notwithstanding, any service that allows people to +run code online is inherently risky. For this reason, I can't make any +guarantees about the security or privacy of your data. + +Please see [Reporting a security issue](SECURITY.md). + +## Can I help? + +Please see [Contributing guide](CONTRIBUTING.md). From d41d8cb55ee93a1ca4009a5df12a284bfef649b8 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 21:52:06 +0000 Subject: [PATCH 045/104] Install specific version of Prettier --- package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/package.json b/package.json index 6af7b34..95ed64d 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "node-pty": "^0.9.0", "p-queue": "^6.6.2", "parse-passwd": "^1.0.0", + "prettier": "^2.3.1", "regenerator-runtime": "^0.13.7", "shell-quote": "^1.7.2", "strip-ansi": "^6.0.0", diff --git a/yarn.lock b/yarn.lock index 3d37d81..08766f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3487,6 +3487,11 @@ postcss@^8.2.15: nanoid "^3.1.23" source-map-js "^0.6.2" +prettier@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.1.tgz#76903c3f8c4449bc9ac597acefa24dc5ad4cbea6" + integrity sha512-p+vNbgpLjif/+D+DwAZAbndtRrR0md0MwfmOVN9N+2RgyACMT+7tfaRnT+WDPkqnuVwleyuBIG2XBxKDme3hPA== + process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" From 3299bd2ddbe7edcfb9089c5f9430ea44724de586 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 21:52:20 +0000 Subject: [PATCH 046/104] Fix misc GBS errors --- Makefile | 7 ++++- tools/generate-build-script.js | 50 +++++++++++++++++----------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 85f2e2f..27ac8f7 100644 --- a/Makefile +++ b/Makefile @@ -101,12 +101,17 @@ repkg: script # L= T= : Build fresh .deb and install into live conta script: # L= T= : Generate a packaging script @: $${L} $${T} +ifeq ($(T),install) + @echo >&2 "use 'make script L=$(L) T=lang' instead" + @exit 1 +endif mkdir -p $(BUILD) node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash + chmod +x $(BUILD)/build.bash ifeq ($(T),lang) node tools/generate-build-script.js --lang $(L) --type install > $(BUILD)/install.bash + chmod +x $(BUILD)/install.bash endif - chmod +x $(BUILD)/build.bash all-scripts: # Generate packaging scripts for all languages node tools/write-all-build-scripts.js diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index 5779e67..f52335f 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -22,31 +22,27 @@ function makeLangScript(langConfig, isShared) { let parts = []; let depends = []; const dependsCfg = (install && install.depends) || {}; - let needsAptGetUpdate = false; + let prefaceNeedsAptGetUpdate = false; + let prepareNeedsAptGetUpdate = false; if ( install && ((install.prepare && - ((install.prepare.manual && install.prepare.manual.includes("apt-get")) || - (install.prepare.apt && install.prepare.apt.length > 0))) || - (install.apt && - install.apt.filter((pkg) => pkg.includes("$")).length > 0)) - ) { - parts.push(`\ -export DEBIAN_FRONTEND=noninteractive`); - if ( - install.prepare && ((install.prepare.manual && install.prepare.manual.includes("apt-get") && install.prepare.manual.includes(":i386")) || (install.prepare.apt && install.prepare.apt.filter((pkg) => pkg.includes(":i386")).length > - 0)) - ) { - parts.push(`\ + 0))) || + (install.preface && + ((install.preface.manual && + install.preface.manual.includes("apt-get") && + install.preface.manual.includes(":i386")) || + (install.preface.apt && + install.preface.apt.filter((pkg) => pkg.includes(":i386")).length > + 0)))) + ) { + prefaceParts.push(`\ dpkg --add-architecture i386`); - } - parts.push(`\ - sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); } if (install) { const { @@ -109,7 +105,7 @@ ${aptRepo.join("\n")} EOF`); } if (apt && apt.length > 0) { - needsAptGetUpdate = true; + prefaceNeedsAptGetUpdate = true; prefaceParts.push(`\ sudo --preserve-env=DEBIAN_FRONTEND apt-get install -y ${apt.join(" ")}`); } @@ -286,7 +282,7 @@ chmod +x "${path}"`); } if (apt) { if (apt.filter((pkg) => pkg.includes("$")).length > 0) { - needsAptGetUpdate = true; + prepareNeedsAptGetUpdate = true; } depends = depends.concat(apt); } @@ -302,10 +298,12 @@ chmod +x "${path}"`); ); } } - if (needsAptGetUpdate) { + if (prefaceNeedsAptGetUpdate) { prefaceParts.unshift(`\ -export DEBIAN_FRONTEND=noninteractive`); - prefaceParts.push(`\ +sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); + } + if (prepareNeedsAptGetUpdate) { + parts.unshift(`\ sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); } parts = prefaceParts.concat(parts); @@ -348,7 +346,9 @@ fi`); parts.unshift(`\ #!/usr/bin/env bash -set -euxo pipefail`); +set -euxo pipefail + +export DEBIAN_FRONTEND=noninteractive`); return parts.join("\n\n"); } @@ -366,7 +366,7 @@ function makeSharedScript(langConfig) { // with its shared dependencies, if any). function makeInstallScript(langConfig) { let parts = []; - const { install } = langConfig; + const { id, install } = langConfig; if (install) { const { apt, cert, aptKey, aptRepo } = install; if (apt && apt.filter((pkg) => pkg.includes(":i386")).length > 0) { @@ -390,7 +390,7 @@ dpkg --add-architecture i386`); .map((src) => { if (src.startsWith("http://") || src.startsWith("https://")) { return `curl -fsSL "${src}" | sudo apt-key add -`; - } else if (/^[0-9A-F]+$/.match(src)) { + } else if (src.match(/^[0-9A-F]+$/)) { return `sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${src}"`; } else { throw new Error(`unknown aptKey format: ${src}`); @@ -416,7 +416,7 @@ export async function generateBuildScript({ lang, type }) { const scriptMaker = { lang: async () => makeLangScript(await readLangConfig(lang)), shared: async () => makeSharedScript(await readSharedDepConfig(lang)), - install: async () => await makeInstallScript(lang), + install: async () => makeInstallScript(await readLangConfig(lang)), }[type]; if (!scriptMaker) { throw new Error(`unsupported script type ${type}`); From 888aea874add8f06469e12eecb3ea8946681f1ea Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 22:01:45 +0000 Subject: [PATCH 047/104] Makefile fixups --- Makefile | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 27ac8f7..422a940 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,9 @@ VOLUME_MOUNT ?= $(PWD) P1 ?= 6119 P2 ?= 6120 -ifneq (,$(E)) +ifneq (,$(EE)) +SHELL_PORTS := -p 0.0.0.0:$(P1):6119 -p 0.0.0.0:$(P2):6120 +else ifneq (,$(E)) SHELL_PORTS := -p 127.0.0.1:$(P1):6119 -p 127.0.0.1:$(P2):6120 else SHELL_PORTS := @@ -145,13 +147,6 @@ pkg-deb: # L= T= [Z=gzip|xz] : Build .deb from packaging environment pkg: pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb -### Install packages - -install: # L= T= : Install built .deb - @: $${L} $${T} - if [[ -z "$$(ls -A /var/lib/apt/lists)" ]]; then sudo apt update; fi - DEBIAN_FRONTEND=noninteractive sudo -E apt reinstall -y ./$(BUILD)/$(DEB) - ### Build and run application code frontend: # Compile frontend assets for production From 1edf4e289aa4f6cad5b3813490c2f75a59479683 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 22:34:12 +0000 Subject: [PATCH 048/104] Privilege escalation --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 422a940..dc34ee2 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,7 @@ all-scripts: # Generate packaging scripts for all languages pkg-clean: # L= T= : Set up fresh packaging environment @: $${L} $${T} - rm -rf $(BUILD)/src $(BUILD)/pkg + sudo rm -rf $(BUILD)/src $(BUILD)/pkg mkdir -p $(BUILD)/src $(BUILD)/pkg pkg-build: # L= T= : Run packaging script in packaging environment From 64fc96ef0626f7d9f6fa30017d9b971a353dac9a Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 22:34:27 +0000 Subject: [PATCH 049/104] Ada language server no longer available as binary This will be recorded on the issue tracker, but for now it seems very difficult to compile the thing from source. --- langs/ada.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/langs/ada.yaml b/langs/ada.yaml index 4b381b2..88fe168 100644 --- a/langs/ada.yaml +++ b/langs/ada.yaml @@ -29,13 +29,6 @@ info: install: apt: - gnat - manual: | - wget https://dl.bintray.com/reznikmm/ada-language-server/linux-latest.tar.gz - tar -xf linux-latest.tar.gz - install -d "${pkg}/usr/local/bin" - install -d "${pkg}/usr/local/lib/x86_64-linux-gnu" - mv linux/ada_language_server "${pkg}/usr/local/bin/ada_language_server" - mv linux/*.so* "${pkg}/usr/local/lib/x86_64-linux-gnu/" main: "main.adb" template: | @@ -50,10 +43,3 @@ compile: | x86_64-linux-gnu-gnatmake-9 main.adb run: | ./main - -lsp: - start: | - ada_language_server - code: "\n Ada.IO" - after: ");" - item: "IO_Exceptions" From b132840385bd09bc34082cc0e8a9f231deca3dd1 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 22:34:53 +0000 Subject: [PATCH 050/104] shell-quote is full of bugs -_- seriously, how did this happen?? this is the one kind of library you'd assume couldn't possibly have bugs, because it's so obviously critical for the thing to be thoroughly tested. and it's not even complicated functionality to implement... --- backend/lsp-repl.js | 4 ++-- backend/sandbox.js | 13 +++---------- backend/util.js | 5 ++++- package.json | 1 - yarn.lock | 5 ----- 5 files changed, 9 insertions(+), 19 deletions(-) diff --git a/backend/lsp-repl.js b/backend/lsp-repl.js index b136110..ed91211 100644 --- a/backend/lsp-repl.js +++ b/backend/lsp-repl.js @@ -2,10 +2,10 @@ import child_process from "child_process"; import process from "process"; import readline from "historic-readline"; -import { quote } from "shell-quote"; import rpc from "vscode-jsonrpc"; import { langsPromise } from "./langs.js"; +import { quote } from "./util.js"; const args = process.argv.slice(2); @@ -32,7 +32,7 @@ if (args.length === 1 && langs[args[0]] && langs[args[0]].lsp) { cmdline = args; } -console.error(quote(cmdline)); +console.error(cmdline.map(quote).join(" ")); const proc = child_process.spawn(cmdline[0], cmdline.slice(1)); proc.stderr.on("data", (data) => process.stderr.write(data)); diff --git a/backend/sandbox.js b/backend/sandbox.js index 19fb2fd..4f8e470 100644 --- a/backend/sandbox.js +++ b/backend/sandbox.js @@ -3,7 +3,6 @@ import { promises as fs } from "fs"; import process from "process"; import pty from "node-pty"; -import { quote } from "shell-quote"; import { readLangConfig } from "../lib/yaml.js"; import { @@ -13,6 +12,7 @@ import { privilegedPty, privilegedSession, privilegedWait, + quote, run, } from "./util.js"; @@ -39,19 +39,12 @@ async function main() { name: "xterm-color", }); await run(privilegedWait({ uuid }), log); - console.log( - bash( - `env L='${lang}' LANG_CONFIG=${quote([ - JSON.stringify(langConfig), - ])} bash --rcfile <(cat <<< ${quote([sandboxScript])})` - )[2] - ); const args = privilegedPty( { uuid }, bash( - `env L='${lang}' LANG_CONFIG=${quote([ + `env L='${lang}' LANG_CONFIG=${quote( JSON.stringify(langConfig), - ])} bash --rcfile <(cat <<< ${quote([sandboxScript])})` + )} bash --rcfile <(cat <<< ${quote(sandboxScript)})` ) ); const proc = spawn(args[0], args.slice(1), { diff --git a/backend/util.js b/backend/util.js index c2f5504..5a205a2 100644 --- a/backend/util.js +++ b/backend/util.js @@ -2,9 +2,12 @@ import { spawn, spawnSync } from "child_process"; import os from "os"; import process from "process"; -import { quote } from "shell-quote"; import { v4 as getUUIDOrig } from "uuid"; +export function quote(str) { + return "'" + str.replace(/'/g, `'"'"'`) + "'"; +} + export const rijuSystemPrivileged = "system/out/riju-system-privileged"; export function getUUID() { diff --git a/package.json b/package.json index 95ed64d..fd543cf 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "parse-passwd": "^1.0.0", "prettier": "^2.3.1", "regenerator-runtime": "^0.13.7", - "shell-quote": "^1.7.2", "strip-ansi": "^6.0.0", "style-loader": "^2.0.0", "uuid": "^8.3.2", diff --git a/yarn.lock b/yarn.lock index 08766f3..5e40894 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3939,11 +3939,6 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shell-quote@^1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2" - integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== - signal-exit@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" From ec0b90dc9b2b07b2f88b06d4d860c20ccc9307eb Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 13 Jun 2021 22:38:41 +0000 Subject: [PATCH 051/104] Snazzier shell prompt --- system/src/riju-system-privileged.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 7614c84..e7546e0 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -45,17 +45,19 @@ char *parseLang(char *lang) { void session(char *uuid, char *lang) { - char *image, *container; + char *image, *container, *hostname; if (asprintf(&image, "riju:lang-%s", lang) < 0) die("asprintf failed"); if (asprintf(&container, "riju-session-%s", uuid) < 0) die("asprintf failed"); + if (asprintf(&hostname, "HOSTNAME=%s", lang) < 0) + die("asprintf failed"); char *argv[] = { "docker", "run", "--rm", "-it", "-e", "HOME=/home/riju", - "-e", "HOSTNAME=riju", + "-e", hostname, "-e", "LANG=C.UTF-8", "-e", "LC_ALL=C.UTF-8", "-e", "LOGNAME=riju", @@ -66,7 +68,7 @@ void session(char *uuid, char *lang) "-e", "TMPDIR=/tmp", "-e", "USER=riju", "-e", "USERNAME=riju", - "--hostname", "riju", + "--hostname", lang, "--name", container, image, "cat", NULL, }; From 4e86dfc5671014582fd7d1e250c88731b0cf6470 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 14 Jun 2021 16:24:16 +0000 Subject: [PATCH 052/104] Add missing return --- backend/server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/server.js b/backend/server.js index 88e54f4..c1a6bc4 100644 --- a/backend/server.js +++ b/backend/server.js @@ -52,6 +52,7 @@ app.get("/:lang", (req, res) => { const canonical = aliases[lang]; if (!canonical) { res.status(404).send(`No such language: ${lang}\n`); + return; } else if (canonical !== lang) { res.redirect(301, `/${canonical}`); return; From ddb2fa0d4b8578587b688a9e916f4822e5a75ec2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 15 Jun 2021 05:11:44 +0000 Subject: [PATCH 053/104] Hallelujah --- Makefile | 18 ++-- backend/api.js | 29 +++++-- backend/sandbox.js | 18 +++- backend/util.js | 4 - docker/admin/Dockerfile | 2 +- docker/app/Dockerfile | 2 +- docker/base/Dockerfile | 2 +- docker/ci/Dockerfile | 2 +- docker/packaging/Dockerfile | 2 +- docker/runtime/Dockerfile | 2 +- docker/shared/admin-pid1.bash | 8 +- system/src/riju-system-privileged.c | 129 +++++++++++++++------------- 12 files changed, 121 insertions(+), 97 deletions(-) diff --git a/Makefile b/Makefile index dc34ee2..1a607c6 100644 --- a/Makefile +++ b/Makefile @@ -73,32 +73,24 @@ endif IMAGE_HASH := -e RIJU_IMAGE_HASH="$$(docker inspect riju:$(LANG_TAG) | jq '.[0].Config.Labels["riju.image-hash"]' -r)" -shell: # I= [L=] [E=1] [P1|P2=] : Launch Docker image with shell +shell: # I= [L=] [E[E]=1] [P1|P2=] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) @mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/run/riju/.aws -v $(HOME)/.docker:/var/run/riju/.docker -v $(HOME)/.ssh:/var/run/riju/.ssh -v $(HOME)/.terraform.d:/var/run/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) - docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) else ifneq (,$(filter $(I),base lang)) ifeq ($(I),lang) @: $${L} endif - docker run -it --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) + docker run -it --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) else ifeq ($(I),runtime) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) else docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) endif -## This is equivalent to 'make pkg' in a fresh packaging container -## followed by 'make install' in a persistent runtime container. - -repkg: script # L= T= : Build fresh .deb and install into live container - @: $${L} $${T} - $(MAKE_QUIETLY) shell I=packaging CMD="make pkg L=$(L) T=$(T)" - ctr="$$(docker container ls -f label="riju-install-target=yes" -l -q)"; test "$${ctr}" || (echo "no valid container is live"; exit 1); docker exec "$${ctr}" make install L=$(L) T=$(T) - ### Build packaging scripts script: # L= T= : Generate a packaging script diff --git a/backend/api.js b/backend/api.js index a39fa23..6030037 100644 --- a/backend/api.js +++ b/backend/api.js @@ -48,7 +48,6 @@ export class Session { }; privilegedSession = () => util.privilegedSession(this.context); - privilegedWait = () => util.privilegedWait(this.context); privilegedExec = (cmdline) => util.privilegedExec(this.context, bash(cmdline)); privilegedPty = (cmdline) => @@ -64,13 +63,6 @@ export class Session { this.container = { pty: containerPty, }; - containerPty.on("data", (data) => - this.send({ - event: "serviceLog", - service: "container", - output: data.toString("utf8"), - }) - ); containerPty.on("close", (code, signal) => this.send({ event: "serviceFailed", @@ -85,7 +77,26 @@ export class Session { error: `${err}`, }) ); - await this.run(this.privilegedWait(this.context)); + let buffer = ""; + await new Promise((resolve) => { + containerPty.on("data", (data) => { + buffer += data; + let idx; + while ((idx = buffer.indexOf("\r\n")) !== -1) { + const line = buffer.slice(0, idx); + buffer = buffer.slice(idx + 2); + if (line === "riju: container ready") { + resolve(); + } else { + this.send({ + event: "serviceLog", + service: "container", + output: line + "\n", + }) + } + } + }); + }); if (this.config.setup) { await this.run(this.privilegedExec(this.config.setup)); } diff --git a/backend/sandbox.js b/backend/sandbox.js index 4f8e470..e0fc67a 100644 --- a/backend/sandbox.js +++ b/backend/sandbox.js @@ -11,7 +11,6 @@ import { privilegedExec, privilegedPty, privilegedSession, - privilegedWait, quote, run, } from "./util.js"; @@ -38,7 +37,22 @@ async function main() { const session = pty.spawn(sessionArgs[0], sessionArgs.slice(1), { name: "xterm-color", }); - await run(privilegedWait({ uuid }), log); + let buffer = ""; + await new Promise((resolve) => { + session.on("data", (data) => { + buffer += data; + let idx; + while ((idx = buffer.indexOf("\r\n")) !== -1) { + const line = buffer.slice(0, idx); + buffer = buffer.slice(idx + 2); + if (line === "riju: container ready") { + resolve(); + } else { + console.error(line); + } + } + }); + }); const args = privilegedPty( { uuid }, bash( diff --git a/backend/util.js b/backend/util.js index 5a205a2..52a2b2a 100644 --- a/backend/util.js +++ b/backend/util.js @@ -51,10 +51,6 @@ export function privilegedSession({ uuid, lang }) { return [rijuSystemPrivileged, "session", uuid, lang]; } -export function privilegedWait({ uuid }) { - return [rijuSystemPrivileged, "wait", uuid]; -} - export function privilegedExec({ uuid }, args) { return [rijuSystemPrivileged, "exec", uuid].concat(args); } diff --git a/docker/admin/Dockerfile b/docker/admin/Dockerfile index 12ee1dc..ea76193 100644 --- a/docker/admin/Dockerfile +++ b/docker/admin/Dockerfile @@ -6,5 +6,5 @@ RUN /tmp/install.bash WORKDIR /src COPY docker/shared/my_init /usr/local/sbin/ COPY docker/shared/admin-pid1.bash /usr/local/sbin/pid1.bash -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--", "/usr/local/sbin/pid1.bash"] +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--skip-runit", "--", "/usr/local/sbin/pid1.bash"] CMD ["bash"] diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 7b8a710..b970056 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -24,7 +24,7 @@ COPY langs ./langs/ FROM riju:runtime -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--skip-runit","--"] RUN useradd -p '!' -m -l -s /usr/bin/bash riju COPY --chown=riju:riju --from=build /src ./ RUN chown root:riju system/out/*-privileged && chmod a=,g=rx,u=rwxs system/out/*-privileged diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 9d23860..60e18e5 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -8,5 +8,5 @@ RUN runuser -u riju -- mkdir /home/riju/src WORKDIR /home/riju/src COPY docker/shared/my_init /usr/local/sbin/ -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"] +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--skip-runit", "--"] CMD ["bash"] diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index 19389e9..8098cb4 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -6,5 +6,5 @@ RUN /tmp/install.bash WORKDIR /src COPY docker/shared/my_init /usr/local/sbin/ COPY docker/shared/admin-pid1.bash /usr/local/sbin/pid1.bash -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--", "/usr/local/sbin/pid1.bash"] +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--skip-runit", "--", "/usr/local/sbin/pid1.bash"] CMD ["bash"] diff --git a/docker/packaging/Dockerfile b/docker/packaging/Dockerfile index 899afb8..ba528b1 100644 --- a/docker/packaging/Dockerfile +++ b/docker/packaging/Dockerfile @@ -5,5 +5,5 @@ RUN /tmp/install.bash WORKDIR /src COPY docker/shared/my_init docker/packaging/pid1.bash /usr/local/sbin/ -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--", "/usr/local/sbin/pid1.bash"] +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--skip-runit", "--", "/usr/local/sbin/pid1.bash"] CMD ["bash"] diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile index 0ac0212..ac1118d 100644 --- a/docker/runtime/Dockerfile +++ b/docker/runtime/Dockerfile @@ -4,7 +4,7 @@ COPY docker/runtime/install.bash /tmp/ RUN /tmp/install.bash COPY docker/shared/my_init docker/runtime/pid1.bash /usr/local/sbin/ -ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--", "/usr/local/sbin/pid1.bash"] +ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--skip-runit", "--", "/usr/local/sbin/pid1.bash"] WORKDIR /src CMD ["bash"] diff --git a/docker/shared/admin-pid1.bash b/docker/shared/admin-pid1.bash index 6fee836..e126a75 100755 --- a/docker/shared/admin-pid1.bash +++ b/docker/shared/admin-pid1.bash @@ -7,10 +7,10 @@ tee -a /etc/hosts >/dev/null <<< "127.0.0.1 $(hostname)" groupadd -g "$(stat -c %g "$PWD")" -o -p '!' -r riju useradd -u "$(stat -c %u "$PWD")" -g "$(stat -c %g "$PWD")" -o -p '!' -m -N -l -s /usr/bin/bash -G sudo riju -runuser -u riju -- ln -sT /var/riju/.aws /home/riju/.aws -runuser -u riju -- ln -sT /var/riju/.docker /home/riju/.docker -runuser -u riju -- ln -sT /var/riju/.ssh /home/riju/.ssh -runuser -u riju -- ln -sT /var/riju/.terraform.d /home/riju/.terraform.d +runuser -u riju -- ln -sT /var/run/riju/.aws /home/riju/.aws +runuser -u riju -- ln -sT /var/run/riju/.docker /home/riju/.docker +runuser -u riju -- ln -sT /var/run/riju/.ssh /home/riju/.ssh +runuser -u riju -- ln -sT /var/run/riju/.terraform.d /home/riju/.terraform.d runuser -u riju -- touch /home/riju/.sudo_as_admin_successful runuser -u riju -- tee -a /home/riju/.bashrc >/dev/null <<"EOF" diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index e7546e0..50ace35 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -1,4 +1,5 @@ #define _GNU_SOURCE +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include #include #include @@ -21,7 +23,6 @@ void die_with_usage() { die("usage:\n" " riju-system-privileged session UUID LANG\n" - " riju-system-privileged wait UUID\n" " riju-system-privileged exec UUID CMDLINE...\n" " riju-system-privileged pty UUID CMDLINE..."); } @@ -43,72 +44,88 @@ char *parseLang(char *lang) { return lang; } -void session(char *uuid, char *lang) -{ - char *image, *container, *hostname; - if (asprintf(&image, "riju:lang-%s", lang) < 0) - die("asprintf failed"); - if (asprintf(&container, "riju-session-%s", uuid) < 0) - die("asprintf failed"); - if (asprintf(&hostname, "HOSTNAME=%s", lang) < 0) - die("asprintf failed"); - char *argv[] = { - "docker", - "run", - "--rm", "-it", - "-e", "HOME=/home/riju", - "-e", hostname, - "-e", "LANG=C.UTF-8", - "-e", "LC_ALL=C.UTF-8", - "-e", "LOGNAME=riju", - "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin", - "-e", "PWD=/home/riju/src", - "-e", "SHELL=/usr/bin/bash", - "-e", "TERM=xterm-256color", - "-e", "TMPDIR=/tmp", - "-e", "USER=riju", - "-e", "USERNAME=riju", - "--hostname", lang, - "--name", container, - image, "cat", NULL, - }; - execvp(argv[0], argv); - die("execvp failed"); -} - void wait_alarm(int signum) { (void)signum; die("container did not come up within 1 second"); } -void wait(char *uuid) +void session(char *uuid, char *lang) { - char *cmdline; - if (asprintf(&cmdline, "docker inspect riju-session-%s >/dev/null 2>&1", uuid) < 0) + char *image, *container, *hostname, *volume, *fifo; + if (asprintf(&image, "riju:lang-%s", lang) < 0) die("asprintf failed"); - struct timespec ts; + if (asprintf(&container, "riju-session-%s", uuid) < 0) + die("asprintf failed"); + if (asprintf(&hostname, "HOSTNAME=%s", lang) < 0) + die("asprintf failed"); + int rv = mkdir("/var/run/riju/sentinels", 0700); + if (rv < 0 && errno != EEXIST) + die("mkdir failed"); + char tmpdir[] = "/var/run/riju/sentinels/XXXXXX"; + if (mkdtemp(tmpdir) == NULL) + die("mkdtemp failed"); + if (asprintf(&volume, "%s:/var/run/riju/sentinel", tmpdir) < 0) + die("asprintf failed"); + if (asprintf(&fifo, "%s/fifo", tmpdir) < 0) + die("asprintf failed"); + if (mknod(fifo, 0700 | S_IFIFO, 0) < 0) + die("mknod failed"); + pid_t pid = fork(); + if (pid < 0) + die("fork failed"); + else if (pid == 0) { + char *argv[] = { + "docker", + "run", + "--rm", + "-v", volume, + "-e", "HOME=/home/riju", + "-e", hostname, + "-e", "LANG=C.UTF-8", + "-e", "LC_ALL=C.UTF-8", + "-e", "LOGNAME=riju", + "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin", + "-e", "PWD=/home/riju/src", + "-e", "SHELL=/usr/bin/bash", + "-e", "TERM=xterm-256color", + "-e", "TMPDIR=/tmp", + "-e", "USER=riju", + "-e", "USERNAME=riju", + "--user", "root", + "--hostname", lang, + "--name", container, + image, "cat", "/var/run/riju/sentinel/fifo", NULL, + }; + execvp(argv[0], argv); + die("execvp failed"); + } + struct timespec ts; // 10ms ts.tv_sec = 0; ts.tv_nsec = 1000 * 1000 * 10; signal(SIGALRM, wait_alarm); alarm(1); + int fd; while (1) { - FILE *proc = popen(cmdline, "r"); - if (proc == NULL) - die("popen failed"); - char buf[1024]; - while (fgets(buf, 1024, proc) != NULL); - if (ferror(proc)) - die("fgets failed"); - int status = pclose(proc); - if (status < 0) - die("pclose failed"); - if (WEXITSTATUS(status) == 0) + fd = open(fifo, O_WRONLY); + if (fd >= 0) break; + if (errno != ENXIO) + die("open failed"); int rv = nanosleep(&ts, NULL); - if (rv != 0 && rv != EINTR) + if (rv != 0 && errno != EINTR) die("nanosleep failed"); } + signal(SIGALRM, SIG_IGN); + if (unlink(fifo) < 0) + die("unlink failed"); + if (rmdir(tmpdir) < 0) + die("rmdir failed"); + printf("riju: container ready\n"); // magic string + if (waitpid(pid, NULL, 0) <= 0) + die("waitpid failed"); + if (close(fd) < 0) + die("close failed"); } void exec(char *uuid, int argc, char **cmdline, bool pty) @@ -119,6 +136,7 @@ void exec(char *uuid, int argc, char **cmdline, bool pty) char *argvPrefix[] = { "docker", "exec", + "--user", "riju", pty ? "-it" : "-i", container, }; @@ -134,8 +152,8 @@ void exec(char *uuid, int argc, char **cmdline, bool pty) int main(int argc, char **argv) { - if (setuid(0) != 0) - die("setuid failed"); + if (seteuid(0) != 0) + die("seteuid failed"); if (argc < 2) die_with_usage(); if (!strcmp(argv[1], "session")) { @@ -146,13 +164,6 @@ int main(int argc, char **argv) session(uuid, lang); return 0; } - if (!strcmp(argv[1], "wait")) { - if (argc != 3) - die_with_usage(); - char *uuid = parseUUID(argv[2]); - wait(uuid); - return 0; - } if (!strcmp(argv[1], "exec")) { if (argc < 4) die_with_usage(); From 491eb12e4ceb011059a2fb1f659587e9479b1b98 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 15 Jun 2021 05:13:47 +0000 Subject: [PATCH 054/104] Fix gnatmake usage in Ada --- langs/ada.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/ada.yaml b/langs/ada.yaml index 88fe168..2572751 100644 --- a/langs/ada.yaml +++ b/langs/ada.yaml @@ -40,6 +40,6 @@ template: | end Main; compile: | - x86_64-linux-gnu-gnatmake-9 main.adb + gnatmake main.adb run: | ./main From ddd5f44d0a0be9ccea030161072f0ceb61fc5a7d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:00:57 +0000 Subject: [PATCH 055/104] Refactor and fixup generate-build-script API --- Makefile | 12 +---------- tools/generate-build-script.js | 36 ++++++++++++++++++++++++-------- tools/write-all-build-scripts.js | 18 +--------------- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 1a607c6..37936ec 100644 --- a/Makefile +++ b/Makefile @@ -95,17 +95,7 @@ endif script: # L= T= : Generate a packaging script @: $${L} $${T} -ifeq ($(T),install) - @echo >&2 "use 'make script L=$(L) T=lang' instead" - @exit 1 -endif - mkdir -p $(BUILD) - node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash - chmod +x $(BUILD)/build.bash -ifeq ($(T),lang) - node tools/generate-build-script.js --lang $(L) --type install > $(BUILD)/install.bash - chmod +x $(BUILD)/install.bash -endif + node tools/generate-build-script.js --lang $(L) --type $(T) all-scripts: # Generate packaging scripts for all languages node tools/write-all-build-scripts.js diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index f52335f..f6bb64d 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -1,3 +1,4 @@ +import { promises as fs } from "fs"; import nodePath from "path"; import process from "process"; import url from "url"; @@ -413,15 +414,32 @@ set -euxo pipefail`); } export async function generateBuildScript({ lang, type }) { - const scriptMaker = { - lang: async () => makeLangScript(await readLangConfig(lang)), - shared: async () => makeSharedScript(await readSharedDepConfig(lang)), - install: async () => makeInstallScript(await readLangConfig(lang)), - }[type]; - if (!scriptMaker) { + const funcs = { + lang: { + cfg: readLangConfig, + make: makeLangScript, + }, + shared: { + cfg: readSharedDepConfig, + make: makeSharedScript, + }, + }; + if (!funcs[type]) { throw new Error(`unsupported script type ${type}`); } - return scriptMaker(); + const { cfg, make } = funcs[type]; + const langConfig = await cfg(lang); + const buildScript = await make(langConfig); + const installScript = await makeInstallScript(langConfig); + await fs.mkdir(`build/${type}/${lang}`, { recursive: true, mode: 0o755 }); + const buildScriptPath = `build/${type}/${lang}/build.bash`; + const installScriptPath = `build/${type}/${lang}/install.bash`; + await Promise.all([ + fs.writeFile(buildScriptPath, buildScript + "\n") + .then(() => fs.chmod(buildScriptPath, 0o755)), + fs.writeFile(installScriptPath, installScript + "\n") + .then(() => fs.chmod(installScriptPath, 0o755)), + ]); } // Parse command-line arguments, run main functionality, and exit. @@ -431,10 +449,10 @@ async function main() { .requiredOption("--lang ", "language ID") .requiredOption( "--type ", - "package category (lang, shared, install)" + "package category (lang or shared)" ); program.parse(process.argv); - console.log(await generateBuildScript(program.opts())); + await generateBuildScript(program.opts()); process.exit(0); } diff --git a/tools/write-all-build-scripts.js b/tools/write-all-build-scripts.js index f03b394..59ad34f 100644 --- a/tools/write-all-build-scripts.js +++ b/tools/write-all-build-scripts.js @@ -12,23 +12,7 @@ import { generateBuildScript } from "./generate-build-script.js"; // Parse command-line arguments, run main functionality, and exit. async function main() { - for (const { lang, type } of await getPackages()) { - const buildScriptPath = `build/${type}/${lang}/build.bash`; - const installScriptPath = `build/${type}/${lang}/install.bash`; - await fs.mkdir(nodePath.dirname(buildScriptPath), { recursive: true }); - await fs.writeFile( - buildScriptPath, - (await generateBuildScript({ lang, type })) + "\n" - ); - await fs.chmod(buildScriptPath, 0o755); - if (type === "lang") { - await fs.writeFile( - installScriptPath, - (await generateBuildScript({ lang, type: "install" })) + "\n" - ); - await fs.chmod(installScriptPath, 0o755); - } - } + await Promise.all((await getPackages()).map(generateBuildScript)); process.exit(0); } From 4ff9d13f556f3bade097b4961702097e341cc453 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:02:13 +0000 Subject: [PATCH 056/104] admin moreutils --- docker/admin/install.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 16d918a..4b4844c 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -42,6 +42,7 @@ jq less make man +moreutils nodejs packer psmisc From 81e78c18e1dba7c8c157e03fb9f338745a6f7bde Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:02:49 +0000 Subject: [PATCH 057/104] Start messing around with ASGs --- tf/infra.tf | 144 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 108 insertions(+), 36 deletions(-) diff --git a/tf/infra.tf b/tf/infra.tf index bf81128..b8ee940 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -11,26 +11,28 @@ terraform { } } -locals { - tags = { - Terraform = "Managed by Terraform" - BillingCategory = "Riju" - } -} - data "external" "env" { program = ["jq", "-n", "env"] } provider "aws" { region = "us-west-1" + default_tags { + tags = { + Terraform = "Managed by Terraform" + BillingCategory = "Riju" + } + } } data "aws_region" "current" {} +data "aws_vpc" "default" { + default = true +} + resource "aws_iam_user" "deploy" { name = "riju-deploy" - tags = local.tags } resource "aws_iam_access_key" "deploy" { @@ -104,7 +106,6 @@ data "aws_iam_policy_document" "riju" { resource "aws_s3_bucket" "riju" { bucket = data.external.env.result.S3_BUCKET - tags = local.tags } resource "aws_s3_bucket_public_access_block" "riju" { @@ -169,42 +170,113 @@ resource "aws_security_group" "server" { protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } - - tags = local.tags } -resource "aws_instance" "server" { - instance_type = "t3.small" - ami = data.aws_ami.server.id - availability_zone = "${data.aws_region.current.name}b" - security_groups = [aws_security_group.server.name] - tags = merge(local.tags, { - Name = "Riju server" - }) - root_block_device { - tags = merge(local.tags, { - Name = "Riju server root volume" - }) +resource "aws_security_group" "alb" { + name = "riju-alb" + description = "Security group for Riju application load balancer" + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTPS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] } } -resource "aws_ebs_volume" "data" { - availability_zone = "${data.aws_region.current.name}b" - size = 125 - type = "gp3" - tags = merge(local.tags, { - Name = "Riju Docker data" - }) +resource "aws_launch_template" "server" { + name = "riju-server" + image_id = data.aws_ami.server.id + instance_type = "t3.small" + security_group_names = [aws_security_group.server.name] + + block_device_mappings { + device_name = "/dev/sdh" + ebs { + volume_type = "gp3" + volume_size = 125 + } + } + + tags = { + Name = "Riju server" + } + + tag_specifications { + resource_type = "instance" + tags = { + Name = "Riju server" + } + } } -resource "aws_volume_attachment" "data" { - device_name = "/dev/sdh" - volume_id = aws_ebs_volume.data.id - instance_id = aws_instance.server.id +resource "aws_autoscaling_group" "server" { + availability_zones = [ + "${data.aws_region.current.name}b", + "${data.aws_region.current.name}c", + ] + desired_capacity = 1 + min_size = 1 + max_size = 3 + + launch_template { + id = aws_launch_template.server.id + } + + tag { + key = "Name" + value = "Riju server" + propagate_at_launch = false + } } -output "server_ip_address" { - value = aws_instance.server.public_ip +resource "aws_lb" "server" { + name = "riju-server" + security_groups = [aws_security_group.alb.name] +} + +resource "aws_lb_target_group" "server_http" { + name = "riju-server-http" + port = 80 + protocol = "HTTP" + vpc_id = data.aws_vpc.default.id +} + +resource "aws_autoscaling_attachment" "server_http" { + autoscaling_group_name = aws_autoscaling_group.server.id + alb_target_group_arn = aws_lb_target_group.server_http.arn +} + +resource "aws_lb_target_group" "server_https" { + name = "riju-server-https" + port = 443 + protocol = "HTTPS" + vpc_id = data.aws_vpc.default.id +} + +resource "aws_autoscaling_attachment" "server_https" { + autoscaling_group_name = aws_autoscaling_group.server.id + alb_target_group_arn = aws_lb_target_group.server_https.arn +} + +output "alb_dns_name" { + value = aws_lb.server } output "deploy_aws_access_key_id" { From b01f1ccab6a8e77b0b841dfef07c2dbd44e28332 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:12:24 +0000 Subject: [PATCH 058/104] Add 'make ecr' target --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 37936ec..b8ba69f 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,9 @@ else docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) endif +ecr: # Authenticate to ECR (temporary credentials) + id="$$(aws sts get-caller-identity | jq .Account -r)"; aws ecr get-login-password --region us-west-1 | docker login --username AWS --password-stdin "$${id}.dkr.ecr.us-west-1.amazonaws.com" + ### Build packaging scripts script: # L= T= : Generate a packaging script From 7375e67c0b821582994ab33de2b8810262f299e1 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:28:01 +0000 Subject: [PATCH 059/104] Fix up ASG subnet shenanigans --- tf/infra.tf | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tf/infra.tf b/tf/infra.tf index b8ee940..b8b0736 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -31,6 +31,15 @@ data "aws_vpc" "default" { default = true } +data "aws_subnet_ids" "default" { + vpc_id = data.aws_vpc.default.id +} + +data "aws_subnet" "default" { + for_each = data.aws_subnet_ids.default.ids + id = each.value +} + resource "aws_iam_user" "deploy" { name = "riju-deploy" } @@ -228,8 +237,7 @@ resource "aws_launch_template" "server" { resource "aws_autoscaling_group" "server" { availability_zones = [ - "${data.aws_region.current.name}b", - "${data.aws_region.current.name}c", + for subnet in data.aws_subnet.default : subnet.availability_zone ] desired_capacity = 1 min_size = 1 @@ -239,6 +247,11 @@ resource "aws_autoscaling_group" "server" { id = aws_launch_template.server.id } + target_group_arns = [ + aws_lb_target_group.server_http.arn, + aws_lb_target_group.server_https.arn, + ] + tag { key = "Name" value = "Riju server" @@ -249,6 +262,7 @@ resource "aws_autoscaling_group" "server" { resource "aws_lb" "server" { name = "riju-server" security_groups = [aws_security_group.alb.name] + subnets = data.aws_subnet_ids.default.ids } resource "aws_lb_target_group" "server_http" { @@ -258,11 +272,6 @@ resource "aws_lb_target_group" "server_http" { vpc_id = data.aws_vpc.default.id } -resource "aws_autoscaling_attachment" "server_http" { - autoscaling_group_name = aws_autoscaling_group.server.id - alb_target_group_arn = aws_lb_target_group.server_http.arn -} - resource "aws_lb_target_group" "server_https" { name = "riju-server-https" port = 443 @@ -270,13 +279,8 @@ resource "aws_lb_target_group" "server_https" { vpc_id = data.aws_vpc.default.id } -resource "aws_autoscaling_attachment" "server_https" { - autoscaling_group_name = aws_autoscaling_group.server.id - alb_target_group_arn = aws_lb_target_group.server_https.arn -} - output "alb_dns_name" { - value = aws_lb.server + value = aws_lb.server.dns_name } output "deploy_aws_access_key_id" { From 8326c7f55824b76ec47e136e845758ef6f04bd51 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:39:59 +0000 Subject: [PATCH 060/104] Fix aptRepo usage --- langs/ceylon.yaml | 2 +- langs/crystal.yaml | 2 +- langs/dart.yaml | 2 +- langs/hack.yaml | 2 +- langs/mongodb.yaml | 2 +- langs/r.yaml | 2 +- tools/jsonschema.yaml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/langs/ceylon.yaml b/langs/ceylon.yaml index f5f427e..3c86586 100644 --- a/langs/ceylon.yaml +++ b/langs/ceylon.yaml @@ -26,7 +26,7 @@ install: aptKey: - "https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key" aptRepo: - - "https://downloads.ceylon-lang.org/apt/ unstable main" + - "deb [arch=amd64] https://downloads.ceylon-lang.org/apt/ unstable main" <<: *add-ceylon-repo apt: - $(grep-aptavail -F Package ceylon -s Package -n | sort -rV | head -n1) diff --git a/langs/crystal.yaml b/langs/crystal.yaml index d2ada3c..bd2e72f 100644 --- a/langs/crystal.yaml +++ b/langs/crystal.yaml @@ -23,7 +23,7 @@ install: aptKey: - "https://keybase.io/crystal/pgp_keys.asc" aptRepo: - - "https://dist.crystal-lang.org/apt crystal main" + - "deb [arch=amd64] https://dist.crystal-lang.org/apt crystal main" apt: - crystal diff --git a/langs/dart.yaml b/langs/dart.yaml index be26adf..076d382 100644 --- a/langs/dart.yaml +++ b/langs/dart.yaml @@ -6,7 +6,7 @@ install: aptKey: - "https://dl-ssl.google.com/linux/linux_signing_key.pub" aptRepo: - - "https://storage.googleapis.com/download.dartlang.org/linux/debian stable main" + - "deb [arch=amd64] https://storage.googleapis.com/download.dartlang.org/linux/debian stable main" apt: - dart diff --git a/langs/hack.yaml b/langs/hack.yaml index d9c65ae..134e187 100644 --- a/langs/hack.yaml +++ b/langs/hack.yaml @@ -9,7 +9,7 @@ install: aptKey: - "B4112585D386EB94" aptRepo: - - "https://dl.hhvm.com/ubuntu ${ubuntu_name} main" + - "deb [arch=amd64] https://dl.hhvm.com/ubuntu ${ubuntu_name} main" apt: - hhvm diff --git a/langs/mongodb.yaml b/langs/mongodb.yaml index 815b814..2e85e92 100644 --- a/langs/mongodb.yaml +++ b/langs/mongodb.yaml @@ -8,7 +8,7 @@ name: "MongoDB" install: prepare: aptRepo: - - "http://archive.ubuntu.com/ubuntu/ focal main universe" + - "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal main universe" manual: | for name in mongodb mongodb-clients mongodb-server mongodb-server-core; do apt-get download "${name}" diff --git a/langs/r.yaml b/langs/r.yaml index 40404da..3137f12 100644 --- a/langs/r.yaml +++ b/langs/r.yaml @@ -8,7 +8,7 @@ install: aptKey: - "E298A3A825C0D65DFD57CBB651716619E084DAB9" aptRepo: - - "https://cloud.r-project.org/bin/linux/ubuntu ${ubuntu_name}-$(curl -fsSL https://cran.r-project.org/bin/linux/ubuntu/ | grep -Eo 'cran[0-9]+' | head -n1)/" + - "deb [arch=amd64] https://cloud.r-project.org/bin/linux/ubuntu ${ubuntu_name}-$(curl -fsSL https://cran.r-project.org/bin/linux/ubuntu/ | grep -Eo 'cran[0-9]+' | head -n1)/" apt: - r-base diff --git a/tools/jsonschema.yaml b/tools/jsonschema.yaml index edc3255..c21c5fd 100644 --- a/tools/jsonschema.yaml +++ b/tools/jsonschema.yaml @@ -533,9 +533,9 @@ properties: type: array items: type: string - pattern: "^https?://" + pattern: "^deb" examples: - - "https://downloads.ceylon-lang.org/apt/ unstable main" + - "deb [arch=amd64] https://downloads.ceylon-lang.org/apt/ unstable main" apt: type: array items: From e87aed83300dbd7aa1b621f76c3cc1220288be71 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:40:20 +0000 Subject: [PATCH 061/104] Get Terraform successfully applied --- tf/infra.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/infra.tf b/tf/infra.tf index b8b0736..da65896 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -261,7 +261,7 @@ resource "aws_autoscaling_group" "server" { resource "aws_lb" "server" { name = "riju-server" - security_groups = [aws_security_group.alb.name] + security_groups = [aws_security_group.alb.id] subnets = data.aws_subnet_ids.default.ids } From df061a1e14913c39b98e3a09b943e7ad4bccac28 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 07:40:42 +0000 Subject: [PATCH 062/104] Tag wine32 with :i386 appropriately --- langs/cmd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/cmd.yaml b/langs/cmd.yaml index 5eef357..9dc4f6b 100644 --- a/langs/cmd.yaml +++ b/langs/cmd.yaml @@ -25,7 +25,7 @@ info: install: apt: - wine - - wine32 + - wine32:i386 repl: | wine cmd From 5c4adfccd03777e202f6c1f7108198e88596ec57 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 08:28:34 +0000 Subject: [PATCH 063/104] Enable 'make sandbox' from admin shell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b8ba69f..3006357 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ shell: # I= [L=] [E[E]=1] [P1|P2=] : Launch Docker image with @: $${I} ifneq (,$(filter $(I),admin ci)) @mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/run/riju/.aws -v $(HOME)/.docker:/var/run/riju/.docker -v $(HOME)/.ssh:/var/run/riju/.ssh -v $(HOME)/.terraform.d:/var/run/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/run/riju/.aws -v $(HOME)/.docker:/var/run/riju/.docker -v $(HOME)/.ssh:/var/run/riju/.ssh -v $(HOME)/.terraform.d:/var/run/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) docker run -it --rm --hostname $(I) -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) else ifneq (,$(filter $(I),base lang)) From b3386e543b3ab2c98f08a6d38798b8d19d80e48f Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 08:28:51 +0000 Subject: [PATCH 064/104] Fix up ASG tagging nonsense --- tf/infra.tf | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tf/infra.tf b/tf/infra.tf index da65896..f02f93a 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -15,13 +15,17 @@ data "external" "env" { program = ["jq", "-n", "env"] } +locals { + tags = { + Terraform = "Managed by Terraform" + BillingCategory = "Riju" + } +} + provider "aws" { region = "us-west-1" default_tags { - tags = { - Terraform = "Managed by Terraform" - BillingCategory = "Riju" - } + tags = local.tags } } @@ -252,11 +256,22 @@ resource "aws_autoscaling_group" "server" { aws_lb_target_group.server_https.arn, ] - tag { - key = "Name" - value = "Riju server" - propagate_at_launch = false - } + tags = concat( + [ + { + key = "Name" + value = "Riju server" + propagate_at_launch = false + } + ], + [ + for key, value in local.tags : { + key = key, + value = value, + propagate_at_launch = true, + } + ], + ) } resource "aws_lb" "server" { From 430550d70466e84a89dd72a763b603249a050009 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 08:38:17 +0000 Subject: [PATCH 065/104] Fix various shenanigans --- langs/factor.yaml | 4 ++-- langs/flex.yaml | 1 + langs/gel.yaml | 4 ++-- langs/r.yaml | 4 ++-- langs/sagemath.yaml | 4 ++-- langs/tcl.yaml | 2 +- langs/zsh.yaml | 4 ++-- 7 files changed, 12 insertions(+), 11 deletions(-) diff --git a/langs/factor.yaml b/langs/factor.yaml index 594b13b..8d6afcc 100644 --- a/langs/factor.yaml +++ b/langs/factor.yaml @@ -14,7 +14,7 @@ install: ln -sT /opt/factor/factor "${pkg}/usr/local/bin/factor-lang" repl: | - factor-lang + HOME="$PWD" factor-lang input: | 123 234 * @@ -27,7 +27,7 @@ template: | createEmpty: "" run: | - factor-lang + HOME="$PWD" factor-lang scope: code: | diff --git a/langs/flex.yaml b/langs/flex.yaml index 803f81b..3cee6cf 100644 --- a/langs/flex.yaml +++ b/langs/flex.yaml @@ -34,4 +34,5 @@ run: | echo 'Reading from stdin, ctrl+D to end input...' >&2 ./main helloInput: | + DELAY: 1 EOF diff --git a/langs/gel.yaml b/langs/gel.yaml index 51f614a..e452b63 100644 --- a/langs/gel.yaml +++ b/langs/gel.yaml @@ -8,7 +8,7 @@ install: - genius repl: | - genius + HOME="$PWD" genius main: ".geniusinit" template: | @@ -16,7 +16,7 @@ template: | createEmpty: "" run: | - genius + HOME="$PWD" genius scope: code: | diff --git a/langs/r.yaml b/langs/r.yaml index 3137f12..472bf96 100644 --- a/langs/r.yaml +++ b/langs/r.yaml @@ -13,14 +13,14 @@ install: - r-base repl: | - R + HOME="$PWD" R main: ".Rprofile" template: | print("Hello, world!") run: | - R --no-save + HOME="$PWD" R --no-save scope: code: | diff --git a/langs/sagemath.yaml b/langs/sagemath.yaml index effa53d..71bf6a3 100644 --- a/langs/sagemath.yaml +++ b/langs/sagemath.yaml @@ -31,7 +31,7 @@ install: ln -s /opt/sagemath/sage "${pkg}/usr/local/bin/" repl: | - sage + HOME="$PWD" sage main: ".sage/init.sage" template: | @@ -39,7 +39,7 @@ template: | createEmpty: "" run: | - sage + HOME="$PWD" sage scope: code: | diff --git a/langs/tcl.yaml b/langs/tcl.yaml index 2d1f5a1..9a61d85 100644 --- a/langs/tcl.yaml +++ b/langs/tcl.yaml @@ -10,7 +10,7 @@ install: - tcl repl: | - tclsh + HOME="$PWD" tclsh input: | expr 123 * 234 diff --git a/langs/zsh.yaml b/langs/zsh.yaml index 214697b..4c2970f 100644 --- a/langs/zsh.yaml +++ b/langs/zsh.yaml @@ -11,7 +11,7 @@ install: - zsh-doc repl: | - SHELL=/usr/bin/zsh zsh + SHELL=/usr/bin/zsh HOME="$PWD" zsh input: | expr 123 \* 234 @@ -21,7 +21,7 @@ template: | createEmpty: "" run: | - SHELL=/usr/bin/zsh zsh + SHELL=/usr/bin/zsh HOME="$PWD" zsh scope: code: | From 9fe7d899b8963b16586c4ca7ed08cbba12bb02b5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 08:41:43 +0000 Subject: [PATCH 066/104] Name ASG reasonably --- tf/infra.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tf/infra.tf b/tf/infra.tf index f02f93a..2929b56 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -240,6 +240,8 @@ resource "aws_launch_template" "server" { } resource "aws_autoscaling_group" "server" { + name = "riju-server" + availability_zones = [ for subnet in data.aws_subnet.default : subnet.availability_zone ] From 6a8c8c1e77f8f5c2f047561af990d2588a9def57 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 08:58:58 +0000 Subject: [PATCH 067/104] Provision TLS certificate via ACM --- tf/infra.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tf/infra.tf b/tf/infra.tf index 2929b56..49e97ba 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -117,6 +117,12 @@ data "aws_iam_policy_document" "riju" { } } +resource "aws_acm_certificate" "riju" { + domain_name = "riju.codes" + subject_alternative_names = ["*.riju.codes"] + validation_method = "DNS" +} + resource "aws_s3_bucket" "riju" { bucket = data.external.env.result.S3_BUCKET } From 46bc2359f36b4eabc3354af3eb860ceb9b1031a2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 09:01:18 +0000 Subject: [PATCH 068/104] Re-add accidentally removed ubuntu_name fn --- tools/generate-build-script.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index f6bb64d..4831547 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -337,6 +337,9 @@ latest_release() { curl -sSL "https://api.github.com/repos/\$1/releases/latest" | jq -r .tag_name }`); } + if (parts.join("\n\n").includes("ubuntu_name")) { + parts.unshift(`ubuntu_name="$(lsb_release -cs)"`); + } if (install && install.disallowCI) { parts.unshift(`\ if [[ -n "\${CI:-}" ]]; then @@ -406,6 +409,9 @@ ${aptRepo.join("\n")} EOF`); } } + if (parts.join("\n\n").includes("ubuntu_name")) { + parts.unshift(`ubuntu_name="$(lsb_release -cs)"`); + } parts.unshift(`\ #!/usr/bin/env bash From 26380534051b2541435146913720b836f9013e67 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 18 Jun 2021 23:10:20 +0000 Subject: [PATCH 069/104] Many misc changes --- Makefile | 25 ++++++++++++++------ backend/api.js | 7 ++---- backend/test-runner.js | 1 - langs/ioke.yaml | 3 +++ langs/mariadb.yaml | 4 +++- langs/mongodb.yaml | 6 ++++- tf/infra.tf | 4 ++++ tools/generate-build-script.js | 42 ++++++++++++++++----------------- tools/generate-deploy-config.js | 24 +++++++++++++++++++ 9 files changed, 80 insertions(+), 36 deletions(-) create mode 100644 tools/generate-deploy-config.js diff --git a/Makefile b/Makefile index 3006357..1cba48f 100644 --- a/Makefile +++ b/Makefile @@ -8,9 +8,10 @@ export BUILD := build/$(T)/$(L) DEB := riju-$(T)-$(L).deb -S3_DEBS := s3://$(S3_BUCKET) -S3_DEB := $(S3_DEBS)/debs/$(DEB) -S3_HASH := $(S3_DEBS)/hashes/riju-$(T)-$(L) +S3 := s3://$(S3_BUCKET) +S3_DEB := $(S3)/debs/$(DEB) +S3_HASH := $(S3)/hashes/riju-$(T)-$(L) +S3_CONFIG := $(S3)/config.json ifneq ($(CMD),) BASH_CMD := bash -c '$(CMD)' @@ -165,7 +166,7 @@ dev: # Compile, run, and watch all artifacts and server for development ## are provided, then only tests matching both are run. test: # [L=[,...]] [T=[,...]] : Run test(s) for language or test category - node backend/test-runner.js $(L) + node backend/test-runner.js ## Functions such as 'repl', 'run', 'format', etc. are available in ## the sandbox, and initial setup has already been done (e.g. 'setup' @@ -185,7 +186,7 @@ lsp: # L= : Run LSP REPL for language or custom command line ### Fetch artifacts from registries -pull: # I= : Pull last published Riju image from Docker Hub +pull: # I= : Pull last published Riju image from Docker registry @: $${I} $${DOCKER_REPO} docker pull $(DOCKER_REPO):$(I) docker tag $(DOCKER_REPO):$(I) riju:$(I) @@ -193,11 +194,15 @@ pull: # I= : Pull last published Riju image from Docker Hub download: # L= T= : Download last published .deb from S3 @: $${L} $${T} $${S3_BUCKET} mkdir -p $(BUILD) - aws s3 cp $(S3_DEB) $(BUILD)/$(DEB) --no-sign-request + aws s3 cp $(S3_DEB) $(BUILD)/$(DEB) + +undeploy: # Pull latest deployment config from S3 + mkdir -p $(BUILD) + aws s3 cp $(S3_CONFIG) $(BUILD)/config.json ### Publish artifacts to registries -push: # I= : Push Riju image to Docker Hub +push: # I= : Push Riju image to Docker registry @: $${I} $${DOCKER_REPO} docker tag riju:$(I) $(DOCKER_REPO):$(I) docker push $(DOCKER_REPO):$(I) @@ -208,6 +213,12 @@ upload: # L= T= : Upload .deb to S3 aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null +config: # Generate deployment config file + node tools/generate-deploy-config.js + +deploy: # Upload deployment config to S3 + aws s3 cp $(BUILD)/config.json $(S3_CONFIG) + ### Miscellaneous ## Run this every time you update .gitignore or .dockerignore.in. diff --git a/backend/api.js b/backend/api.js index 6030037..d722589 100644 --- a/backend/api.js +++ b/backend/api.js @@ -1,5 +1,6 @@ import { spawn } from "child_process"; import path from "path"; +import process from "process"; import WebSocket from "ws"; import pty from "node-pty"; @@ -299,11 +300,7 @@ export class Session { template, } = this.config; if (this.term) { - const pid = this.term.pty.pid; - const args = this.privilegedExec( - `kill -SIGTERM ${pid}; sleep 1; kill -SIGKILL ${pid}` - ); - spawn(args[0], args.slice(1)); + process.kill(this.term.pty.pid); // Signal to terminalOutput message generator using closure. this.term.live = false; this.term = null; diff --git a/backend/test-runner.js b/backend/test-runner.js index b7b60b3..e232a23 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -623,7 +623,6 @@ async function writeLog(lang, type, result, log) { async function main() { langs = await langsPromise; let tests = getTestList(); - const args = process.argv.slice(2); if (process.env.L) { tests = tests.filter(({ lang }) => process.env.L.split().includes(lang)); } diff --git a/langs/ioke.yaml b/langs/ioke.yaml index 2cf5ef7..709b00d 100644 --- a/langs/ioke.yaml +++ b/langs/ioke.yaml @@ -4,6 +4,9 @@ aliases: name: "Ioke" install: + prepare: + cert: + - "https://letsencrypt.org/certs/lets-encrypt-r3.pem" apt: - default-jdk manual: | diff --git a/langs/mariadb.yaml b/langs/mariadb.yaml index 17b21e0..d2faee7 100644 --- a/langs/mariadb.yaml +++ b/langs/mariadb.yaml @@ -10,11 +10,13 @@ install: - mysql-client riju: - sqls + # MariaDB has Debian package downloads, but only for LTS versions of + # Ubuntu, so we have to download the release tarball instead. manual: | install -d "${pkg}/opt/mariadb" ver="$(curl -sSL https://downloads.mariadb.org/ | grep 'href="/mariadb/[0-9]' | grep -Eo '[0-9][^/]+' | sort -rV | head -n1)" - wget "https://downloads.mariadb.org/f/mariadb-${ver}/bintar-linux-x86_64/mariadb-${ver}-linux-x86_64.tar.gz/from/http%3A//sfo1.mirrors.digitalocean.com/mariadb/?serve" -O mariadb.tar.gz + wget "https://downloads.mariadb.org/f/mariadb-${ver}/bintar-linux-systemd-x86_64/mariadb-${ver}-linux-systemd-x86_64.tar.gz/from/http%3A//sfo1.mirrors.digitalocean.com/mariadb/?serve" -O mariadb.tar.gz tar -xf mariadb.tar.gz -C "${pkg}/opt/mariadb" --strip-components=1 chmod a=rx,u=rwx "${pkg}/opt/mariadb/lib/plugin/auth_pam_tool_dir" chmod a=rx,u=rwxs "${pkg}/opt/mariadb/lib/plugin/auth_pam_tool_dir/auth_pam_tool" diff --git a/langs/mongodb.yaml b/langs/mongodb.yaml index 2e85e92..d7a86de 100644 --- a/langs/mongodb.yaml +++ b/langs/mongodb.yaml @@ -6,10 +6,14 @@ aliases: name: "MongoDB" install: + # The MongoDB package is only available for LTS releases of Ubuntu, + # so we grab it from focal. prepare: aptRepo: - "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ focal main universe" manual: | + sudo --preserve-env=DEBIAN_FRONTEND apt-get update + for name in mongodb mongodb-clients mongodb-server mongodb-server-core; do apt-get download "${name}" mv "${name}"_*.deb "${name}.deb" @@ -42,8 +46,8 @@ template: | print("Hello, world!") run: | - set -e while ps -u "$(id -un)" -o comm | grep -q mongod; do + ps -u "$(id -un)" -o pid,comm | cat sleep 0.01 done rm -rf data diff --git a/tf/infra.tf b/tf/infra.tf index 49e97ba..b53c8d8 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -121,6 +121,10 @@ resource "aws_acm_certificate" "riju" { domain_name = "riju.codes" subject_alternative_names = ["*.riju.codes"] validation_method = "DNS" + + tags = { + Name = "Riju server" + } } resource "aws_s3_bucket" "riju" { diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index 4831547..0f9fd08 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -25,26 +25,6 @@ function makeLangScript(langConfig, isShared) { const dependsCfg = (install && install.depends) || {}; let prefaceNeedsAptGetUpdate = false; let prepareNeedsAptGetUpdate = false; - if ( - install && - ((install.prepare && - ((install.prepare.manual && - install.prepare.manual.includes("apt-get") && - install.prepare.manual.includes(":i386")) || - (install.prepare.apt && - install.prepare.apt.filter((pkg) => pkg.includes(":i386")).length > - 0))) || - (install.preface && - ((install.preface.manual && - install.preface.manual.includes("apt-get") && - install.preface.manual.includes(":i386")) || - (install.preface.apt && - install.preface.apt.filter((pkg) => pkg.includes(":i386")).length > - 0)))) - ) { - prefaceParts.push(`\ -dpkg --add-architecture i386`); - } if (install) { const { prepare, @@ -302,6 +282,26 @@ chmod +x "${path}"`); if (prefaceNeedsAptGetUpdate) { prefaceParts.unshift(`\ sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); + } + if ( + install && + ((install.prepare && + ((install.prepare.manual && + install.prepare.manual.includes("apt-get") && + install.prepare.manual.includes(":i386")) || + (install.prepare.apt && + install.prepare.apt.filter((pkg) => pkg.includes(":i386")).length > + 0))) || + (install.preface && + ((install.preface.manual && + install.preface.manual.includes("apt-get") && + install.preface.manual.includes(":i386")) || + (install.preface.apt && + install.preface.apt.filter((pkg) => pkg.includes(":i386")).length > + 0)))) + ) { + prefaceParts.unshift(`\ +sudo dpkg --add-architecture i386`); } if (prepareNeedsAptGetUpdate) { parts.unshift(`\ @@ -375,7 +375,7 @@ function makeInstallScript(langConfig) { const { apt, cert, aptKey, aptRepo } = install; if (apt && apt.filter((pkg) => pkg.includes(":i386")).length > 0) { parts.push(`\ -dpkg --add-architecture i386`); +sudo dpkg --add-architecture i386`); } if (cert && cert.length > 0) { parts.push( diff --git a/tools/generate-deploy-config.js b/tools/generate-deploy-config.js new file mode 100644 index 0000000..b3c6d25 --- /dev/null +++ b/tools/generate-deploy-config.js @@ -0,0 +1,24 @@ +import { promises as fs } from "fs"; +import url from "url"; + +// Get the contents of the JSON file that will be written to S3 in +// order to deploy Riju. +async function getDeployConfig() { + // TODO +} + +// Parse command-line arguments, run main functionality, and exit. +async function main() { + const program = new Command(); + program.parse(process.argv); + await fs.mkdir("build", { recursive: true }); + await fs.writeFile("build/config.json", JSON.stringify(await getDeployConfig(), null, 2) + "\n"); + process.exit(0); +} + +if (process.argv[1] === url.fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} From 9256f228e6f844d41ae210eab6f1152529d0d60d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 19 Jun 2021 08:06:05 +0000 Subject: [PATCH 070/104] Oh god, well this was terrible --- docker/admin/install.bash | 1 + docker/runtime/install.bash | 1 + system/res/docker-exec.py | 85 +++++++++++++++++++++++++++++ system/src/riju-system-privileged.c | 4 +- 4 files changed, 89 insertions(+), 2 deletions(-) create mode 100755 system/res/docker-exec.py diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 4b4844c..7d784b4 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -53,6 +53,7 @@ sudo tmux terraform unzip +uuid-runtime vim wget yarn diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 73283df..011f226 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -59,6 +59,7 @@ strace sudo tmux tree +uuid-runtime vim " diff --git a/system/res/docker-exec.py b/system/res/docker-exec.py new file mode 100755 index 0000000..48b8634 --- /dev/null +++ b/system/res/docker-exec.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +import argparse +import signal +import subprocess +import sys +import uuid + +class Parser(argparse.ArgumentParser): + def format_help(self): + return """ +Usage: docker-exec.bash [OPTIONS] CONTAINER COMMAND [ARG...] + +Run a command in a running container + +Options: + -i, --interactive Keep STDIN open even if not attached + -t, --tty Allocate a pseudo-TTY + -u, --user string Username or UID (format: :[]) +""" + +parser = Parser() +parser.add_argument("-i", "--interactive", action="store_true") +parser.add_argument("-t", "--tty", action="store_true") +parser.add_argument("-u", "--user", type=str) +parser.add_argument("container", type=str) +parser.add_argument("arg", type=str, nargs="*") + +args = parser.parse_args() + +pidfiles = "/var/run/riju/pidfiles" +pidfile = pidfiles + "/" + str(uuid.uuid4()).replace("-", "") +print(pidfile) + +# We have to use 'kill -9' here, otherwise runuser intercepts the +# signal and takes its sweet time cleaning up. +def cleanup(*ignored_args): + subprocess.run([ + "docker", + "exec", + args.container, + "bash", + "-c", + f""" +set -euo pipefail +if [[ -f '{pidfile}' ]]; then + kill -9 -$(< '{pidfile}') + rm -f '{pidfile}' +fi + """ + ]) + +signal.signal(signal.SIGINT, cleanup) +signal.signal(signal.SIGTERM, cleanup) + +exec_args = [] + +if args.interactive: + exec_args.append("-i") +if args.tty: + exec_args.append("-t") + +runuser_args = [] + +if args.user: + runuser_args = ["runuser", "-u", args.user, "--"] + +subprocess.run([ + "docker", + "exec", + *exec_args, + args.container, + "bash", + "-c", + f""" +set -euo pipefail +umask 077 +mkdir -p '{pidfiles}' +echo "$$" > '{pidfile}' +exec "$@" + """, + "--", + *runuser_args, + *args.arg, +]) diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 50ace35..493c9d3 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -134,11 +134,11 @@ void exec(char *uuid, int argc, char **cmdline, bool pty) if (asprintf(&container, "riju-session-%s", uuid) < 0) die("asprintf failed"); char *argvPrefix[] = { - "docker", - "exec", + "./system/res/docker-exec.py", "--user", "riju", pty ? "-it" : "-i", container, + "--", }; char **argv = malloc(sizeof(argvPrefix) + (argc + 1) * sizeof(char *)); if (argv == NULL) From fcf5432058d84c8cba9acae8c7eb576ae4880940 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 19 Jun 2021 08:09:22 +0000 Subject: [PATCH 071/104] debug --- backend/api.js | 6 +++++- system/res/docker-exec.py | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/api.js b/backend/api.js index d722589..d930d25 100644 --- a/backend/api.js +++ b/backend/api.js @@ -300,7 +300,11 @@ export class Session { template, } = this.config; if (this.term) { - process.kill(this.term.pty.pid); + try { + process.kill(this.term.pty.pid); + } catch (err) { + // process might have already exited + } // Signal to terminalOutput message generator using closure. this.term.live = false; this.term = null; diff --git a/system/res/docker-exec.py b/system/res/docker-exec.py index 48b8634..8d13799 100755 --- a/system/res/docker-exec.py +++ b/system/res/docker-exec.py @@ -30,7 +30,6 @@ args = parser.parse_args() pidfiles = "/var/run/riju/pidfiles" pidfile = pidfiles + "/" + str(uuid.uuid4()).replace("-", "") -print(pidfile) # We have to use 'kill -9' here, otherwise runuser intercepts the # signal and takes its sweet time cleaning up. @@ -44,7 +43,7 @@ def cleanup(*ignored_args): f""" set -euo pipefail if [[ -f '{pidfile}' ]]; then - kill -9 -$(< '{pidfile}') + kill -9 -$(< '{pidfile}') 2>/dev/null || true rm -f '{pidfile}' fi """ From 4ca46b0f1f5cfa311367add143027742b9e3c88e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 03:39:34 +0000 Subject: [PATCH 072/104] Fix mumps --- langs/mumps.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langs/mumps.yaml b/langs/mumps.yaml index 9e254f8..a1c6073 100644 --- a/langs/mumps.yaml +++ b/langs/mumps.yaml @@ -16,4 +16,5 @@ template: | quit run: | - gtm_dist=/usr/lib/x86_64-linux-gnu/fis-gtm/V6.3-007_x86_64 /usr/lib/x86_64-linux-gnu/fis-gtm/V6.3-007_x86_64/utf8/mumps -r main main.m + gtm_dist=(/usr/lib/x86_64-linux-gnu/fis-gtm/V*_x86_64) + gtm_dist="${gtm_dist[@]}" "${gtm_dist[@]}/utf8/mumps" -r main main.m From 3b149bb109f454bf29205d908107a2715cba18de Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 03:52:41 +0000 Subject: [PATCH 073/104] Quick change --- tools/depgraph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/depgraph.js b/tools/depgraph.js index 75afee1..96f2555 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -247,7 +247,7 @@ async function getDeployArtifact(langs) { .concat(langs.map((lang) => `test:lang-${lang}`)), publishTarget: true, publishToRegistry: async () => { - await runCommand(`tools/deploy.bash`); + await runCommand(`make deploy`); }, }; } From 566772f58ad10a4c1155915af580f953031767e5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 03:52:49 +0000 Subject: [PATCH 074/104] Fix ocaml --- langs/ocaml.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/langs/ocaml.yaml b/langs/ocaml.yaml index fd3e20c..4864a31 100644 --- a/langs/ocaml.yaml +++ b/langs/ocaml.yaml @@ -6,10 +6,7 @@ install: - ocaml-nox opam: - ocamlformat - - name: ocaml-lsp-server - source: "https://github.com/ocaml/ocaml-lsp.git" - binaries: - - ocamllsp + - ocaml-lsp-server repl: | ocaml From 582e83504e0bcd1a6192052d99c9d07f78331f89 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 04:03:14 +0000 Subject: [PATCH 075/104] Just for now --- tf/infra.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/infra.tf b/tf/infra.tf index b53c8d8..c03f00f 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -255,8 +255,8 @@ resource "aws_autoscaling_group" "server" { availability_zones = [ for subnet in data.aws_subnet.default : subnet.availability_zone ] - desired_capacity = 1 - min_size = 1 + desired_capacity = 0 + min_size = 0 max_size = 3 launch_template { From 7ed020f4ce519d62998d6f51641e9f76c4438486 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 04:13:10 +0000 Subject: [PATCH 076/104] Allow 'terraform apply' without AMI --- tf/infra.tf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tf/infra.tf b/tf/infra.tf index c03f00f..de05600 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -20,6 +20,8 @@ locals { Terraform = "Managed by Terraform" BillingCategory = "Riju" } + + ami_available = lookup(data.external.env.result, "AMI_NAME", "") != "" ? true : false } provider "aws" { @@ -151,6 +153,8 @@ resource "aws_ecr_repository" "riju" { } data "aws_ami" "server" { + count = local.ami_available ? 1 : 0 + owners = ["self"] filter { @@ -224,8 +228,10 @@ resource "aws_security_group" "alb" { } resource "aws_launch_template" "server" { + count = local.ami_available ? 1 : 0 + name = "riju-server" - image_id = data.aws_ami.server.id + image_id = data.aws_ami.server[0].id instance_type = "t3.small" security_group_names = [aws_security_group.server.name] @@ -250,6 +256,8 @@ resource "aws_launch_template" "server" { } resource "aws_autoscaling_group" "server" { + count = local.ami_available ? 1 : 0 + name = "riju-server" availability_zones = [ @@ -260,7 +268,7 @@ resource "aws_autoscaling_group" "server" { max_size = 3 launch_template { - id = aws_launch_template.server.id + id = aws_launch_template.server[0].id } target_group_arns = [ From dd99bdf778b3e52f18c09dcb270d1a92b280bcd7 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 04:36:32 +0000 Subject: [PATCH 077/104] Fix qsharp --- langs/qsharp.yaml | 6 +++++- tools/generate-build-script.js | 11 ++++++++++- tools/jsonschema.yaml | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/langs/qsharp.yaml b/langs/qsharp.yaml index 6155b1f..14c6224 100644 --- a/langs/qsharp.yaml +++ b/langs/qsharp.yaml @@ -9,9 +9,9 @@ install: # 3.x of the .NET SDK. Not sure why. prepare: &install-dotnet preface: | - sudo --preserve-env=DEBIAN_FRONTEND apt-get update wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" sudo --preserve-env=DEBIAN_FRONTEND apt-get install ./packages-microsoft-prod.deb + sudo --preserve-env=DEBIAN_FRONTEND apt-get update apt: - $(grep-aptavail -wF Package "dotnet-sdk-3\.[0-9.]+" -s Package -n | sort -Vr | head -n1) <<: *install-dotnet @@ -33,6 +33,10 @@ install: cp -R * "${HOME}/.dotnet" "${HOME}/.nuget" "${pkg}/opt/qsharp/skel/" rm "${pkg}/opt/qsharp/skel/main/Program.qs" chmod -R a=u,go-w "${pkg}/opt/qsharp/skel" + manualInstall: | + wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" + sudo --preserve-env=DEBIAN_FRONTEND apt-get update + sudo --preserve-env=DEBIAN_FRONTEND apt-get install ./packages-microsoft-prod.deb setup: | shopt -s dotglob diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index 0f9fd08..d594090 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -340,6 +340,9 @@ latest_release() { if (parts.join("\n\n").includes("ubuntu_name")) { parts.unshift(`ubuntu_name="$(lsb_release -cs)"`); } + if (parts.join("\n\n").includes("ubuntu_ver")) { + parts.unshift(`ubuntu_ver="$(lsb_release -rs)"`); + } if (install && install.disallowCI) { parts.unshift(`\ if [[ -n "\${CI:-}" ]]; then @@ -372,7 +375,7 @@ function makeInstallScript(langConfig) { let parts = []; const { id, install } = langConfig; if (install) { - const { apt, cert, aptKey, aptRepo } = install; + const { apt, cert, aptKey, aptRepo, manualInstall } = install; if (apt && apt.filter((pkg) => pkg.includes(":i386")).length > 0) { parts.push(`\ sudo dpkg --add-architecture i386`); @@ -408,10 +411,16 @@ sudo dpkg --add-architecture i386`); ${aptRepo.join("\n")} EOF`); } + if (manualInstall) { + parts.push(manualInstall); + } } if (parts.join("\n\n").includes("ubuntu_name")) { parts.unshift(`ubuntu_name="$(lsb_release -cs)"`); } + if (parts.join("\n\n").includes("ubuntu_ver")) { + parts.unshift(`ubuntu_name="$(lsb_release -rs)"`); + } parts.unshift(`\ #!/usr/bin/env bash diff --git a/tools/jsonschema.yaml b/tools/jsonschema.yaml index c21c5fd..168357d 100644 --- a/tools/jsonschema.yaml +++ b/tools/jsonschema.yaml @@ -627,6 +627,9 @@ properties: manual: type: string minLength: 1 + manualInstall: + type: string + minLength: 1 deb: type: array items: From 4c8bee2c863e366d91e8e524aee7314d43408328 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 04:36:39 +0000 Subject: [PATCH 078/104] Fix depgraph bug --- tools/depgraph.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/depgraph.js b/tools/depgraph.js index 96f2555..65305cd 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -120,11 +120,16 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { } let salt = null; if (isLangImage) { + const installContents = await fs.readFile( + `build/lang/${isLangImage.lang}/install.bash`, + "utf-8" + ); salt = { langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`], sharedHashes: isLangImage.sharedDeps.map( (name) => dependencyHashes[`deb:shared-${name}`] ), + installHash: crypto.createHash("sha1").update(installContents).digest("hex"), }; } return await hashDockerfile(name, dependentDockerHashes, { salt }); From 61cb3d1bea891517cea4d928e225ae45244de1c7 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 05:12:04 +0000 Subject: [PATCH 079/104] For future reference --- tf/.terraform.lock.hcl | 19 +++++++++++++++++++ tf/infra.tf | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/tf/.terraform.lock.hcl b/tf/.terraform.lock.hcl index 14a64fd..2fc3b5a 100644 --- a/tf/.terraform.lock.hcl +++ b/tf/.terraform.lock.hcl @@ -37,3 +37,22 @@ provider "registry.terraform.io/hashicorp/external" { "zh:ddebfd1e674ba336df09b1f27bbaa0e036c25b7a7087dc8081443f6e5954028b", ] } + +provider "registry.terraform.io/hashicorp/null" { + version = "3.1.0" + constraints = "3.1.0" + hashes = [ + "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=", + "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", + "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", + "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", + "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", + "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", + "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", + "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", + "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", + "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", + "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", + "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", + ] +} diff --git a/tf/infra.tf b/tf/infra.tf index de05600..b76b5ce 100644 --- a/tf/infra.tf +++ b/tf/infra.tf @@ -8,6 +8,10 @@ terraform { source = "hashicorp/aws" version = "~> 3.45" } + null = { + source = "hashicorp/null" + version = "~> 3.1" + } } } @@ -129,6 +133,10 @@ resource "aws_acm_certificate" "riju" { } } +resource "aws_acm_certificate_validation" "riju" { + certificate_arn = aws_acm_certificate.riju.arn +} + resource "aws_s3_bucket" "riju" { bucket = data.external.env.result.S3_BUCKET } From e92b2aea4f0ad465efafd9c06de2322395ae7ba2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 05:12:18 +0000 Subject: [PATCH 080/104] Add basic Go build system --- Makefile | 10 ++++++++-- docker/admin/install.bash | 1 + supervisor/compile.bash | 7 +++++++ supervisor/go.mod | 3 +++ supervisor/src/main.go | 9 +++++++++ 5 files changed, 28 insertions(+), 2 deletions(-) create mode 100755 supervisor/compile.bash create mode 100644 supervisor/go.mod create mode 100644 supervisor/src/main.go diff --git a/Makefile b/Makefile index 1cba48f..bf207e9 100644 --- a/Makefile +++ b/Makefile @@ -147,16 +147,22 @@ system: # Compile setuid binary for production system-dev: # Compile and watch setuid binary for development watchexec -w system/src -n -- ./system/compile.bash +supervisor: # Compile supervisor binary for production + ./supervisor/compile.bash + +supervisor-dev: # Compile and watch supervisor binary for development + watchexec -w supervisor/src -n -- ./supervisor/compile.bash + server: # Run server for production node backend/server.js server-dev: # Run and restart server for development watchexec -w backend -r -n -- node backend/server.js -build: frontend system # Compile all artifacts for production +build: frontend system supervisor # Compile all artifacts for production dev: # Compile, run, and watch all artifacts and server for development - $(MAKE_QUIETLY) -j3 frontend-dev system-dev server-dev + $(MAKE_QUIETLY) -j3 frontend-dev system-dev supervisor-dev server-dev ### Application tools diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 7d784b4..3fbaf07 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -37,6 +37,7 @@ dctrl-tools docker-ce-cli g++ git +golang htop jq less diff --git a/supervisor/compile.bash b/supervisor/compile.bash new file mode 100755 index 0000000..1152696 --- /dev/null +++ b/supervisor/compile.bash @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd supervisor +mkdir -p out +go build -o out/riju-supervisor ./src diff --git a/supervisor/go.mod b/supervisor/go.mod new file mode 100644 index 0000000..e766e88 --- /dev/null +++ b/supervisor/go.mod @@ -0,0 +1,3 @@ +module github.com/raxod502/riju/supervisor + +go 1.16 diff --git a/supervisor/src/main.go b/supervisor/src/main.go new file mode 100644 index 0000000..8f61880 --- /dev/null +++ b/supervisor/src/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "fmt" +) + +func main() { + fmt.Println("Hello, world!") +} From 41a14ac1ae8775a2a68e262fc996c9d62f4c9599 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 17:28:04 +0000 Subject: [PATCH 081/104] Oops --- tools/build-lang-image.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js index d9016aa..2fb24c1 100644 --- a/tools/build-lang-image.js +++ b/tools/build-lang-image.js @@ -1,3 +1,4 @@ +import crypto from "crypto"; import { promises as fs } from "fs"; import http from "http"; import url from "url"; @@ -26,6 +27,10 @@ async function main() { program.option("--debug", "interactive debugging"); program.parse(process.argv); const { lang, debug } = program.opts(); + const installContents = await fs.readFile( + `build/lang/${lang}/install.bash`, + "utf-8" + ); const hash = await hashDockerfile( "lang", { @@ -42,6 +47,7 @@ async function main() { ) ) ).sort(), + installHash: crypto.createHash("sha1").update(installContents).digest("hex"), }, } ); From 22dcdee87146f4ecefe1a9a9b364b1573b5d58bc Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 18:43:10 +0000 Subject: [PATCH 082/104] Supervisor gains proxy ability --- Makefile | 2 +- docker/admin/install.bash | 1 + docker/runtime/install.bash | 2 ++ supervisor/compile.bash | 7 +++- supervisor/src/main.go | 71 +++++++++++++++++++++++++++++++++++-- 5 files changed, 79 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index bf207e9..54898e9 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ server-dev: # Run and restart server for development build: frontend system supervisor # Compile all artifacts for production dev: # Compile, run, and watch all artifacts and server for development - $(MAKE_QUIETLY) -j3 frontend-dev system-dev supervisor-dev server-dev + $(MAKE_QUIETLY) -j4 frontend-dev system-dev supervisor-dev server-dev ### Application tools diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 3fbaf07..5bf500a 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -39,6 +39,7 @@ g++ git golang htop +httpie jq less make diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 011f226..9723f42 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -48,7 +48,9 @@ dctrl-tools bind9-dnsutils less git +golang htop +httpie jq make man diff --git a/supervisor/compile.bash b/supervisor/compile.bash index 1152696..52a124a 100755 --- a/supervisor/compile.bash +++ b/supervisor/compile.bash @@ -2,6 +2,11 @@ set -euo pipefail +function verbosely { + echo "$@" + "$@" +} + cd supervisor mkdir -p out -go build -o out/riju-supervisor ./src +verbosely go build -o out/riju-supervisor ./src diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 8f61880..433e4ae 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -2,8 +2,75 @@ package main import ( "fmt" + "log" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync" ) -func main() { - fmt.Println("Hello, world!") +type supervisor struct { + proxyHandler http.Handler + + reloadLock sync.Mutex + reloadInProgress bool + reloadNeeded bool +} + +func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api/supervisor") { + if r.URL.Path == "/api/supervisor/v1/reload" { + resp := "" + sv.reloadLock.Lock() + if !sv.reloadInProgress { + sv.reloadInProgress = true + go sv.reload() + resp = "Triggered reload." + } else { + sv.reloadNeeded = true + resp = "Reload already in progress, scheduling another one." + } + sv.reloadLock.Unlock() + fmt.Fprintln(w, resp) + return + } + http.NotFound(w, r) + return + } + sv.proxyHandler.ServeHTTP(w, r) + return +} + +func (sv *supervisor) reloadWithScheduling() { + err := sv.reload() + if err != nil { + log.Printf("failed to reload: %s\n", err.Error()) + } else { + log.Println("successfully reloaded") + } + sv.reloadLock.Lock() + sv.reloadInProgress = false + if sv.reloadNeeded { + sv.reloadInProgress = true + go sv.reloadWithScheduling() + } + sv.reloadLock.Unlock() +} + +func (sv *supervisor) reload() error { + fmt.Println("starting reload") + return nil +} + +func main() { + url, err := url.Parse("http://localhost:6119") + if err != nil { + log.Fatal(err) + } + sv := &supervisor{ + proxyHandler: httputil.NewSingleHostReverseProxy(url), + } + log.Println("listening on http://0.0.0.0:80") + log.Fatalln(http.ListenAndServe("0.0.0.0:80", sv)) } From 442265a6424972a6b37eb1819dfecf5f040f5840 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Jul 2021 19:09:47 +0000 Subject: [PATCH 083/104] Yeesh --- Makefile | 21 +++++++++++++-------- backend/test-runner.js | 2 +- tools/depgraph.js | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 54898e9..3e5a59e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL := bash .SHELLFLAGS := -o pipefail -euc -export PATH := bin:$(PATH) +export PATH := /src/bin:$(PATH) -include .env export @@ -72,24 +72,27 @@ else LANG_TAG := $(I) endif -IMAGE_HASH := -e RIJU_IMAGE_HASH="$$(docker inspect riju:$(LANG_TAG) | jq '.[0].Config.Labels["riju.image-hash"]' -r)" +IMAGE_HASH := "$$(docker inspect riju:$(LANG_TAG) | jq '.[0].Config.Labels["riju.image-hash"]' -r)" +WITH_IMAGE_HASH := -e RIJU_IMAGE_HASH=$(IMAGE_HASH) + +LANG_IMAGE_HASH := "$$(docker inspect riju:lang-$(L) | jq '.[0].Config.Labels["riju.image-hash"]' -r)" shell: # I= [L=] [E[E]=1] [P1|P2=] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) @mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/run/riju/.aws -v $(HOME)/.docker:/var/run/riju/.docker -v $(HOME)/.ssh:/var/run/riju/.ssh -v $(HOME)/.terraform.d:/var/run/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/run/riju/.aws -v $(HOME)/.docker:/var/run/riju/.docker -v $(HOME)/.ssh:/var/run/riju/.ssh -v $(HOME)/.terraform.d:/var/run/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) - docker run -it --rm --hostname $(I) -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) else ifneq (,$(filter $(I),base lang)) ifeq ($(I),lang) @: $${L} endif - docker run -it --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) + docker run -it --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) else ifeq ($(I),runtime) - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/riju:/var/run/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) else - docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) endif ecr: # Authenticate to ECR (temporary credentials) @@ -172,7 +175,7 @@ dev: # Compile, run, and watch all artifacts and server for development ## are provided, then only tests matching both are run. test: # [L=[,...]] [T=[,...]] : Run test(s) for language or test category - node backend/test-runner.js + RIJU_LANG_IMAGE_HASH=$(LANG_IMAGE_HASH) node backend/test-runner.js ## Functions such as 'repl', 'run', 'format', etc. are available in ## the sandbox, and initial setup has already been done (e.g. 'setup' @@ -210,6 +213,8 @@ undeploy: # Pull latest deployment config from S3 push: # I= : Push Riju image to Docker registry @: $${I} $${DOCKER_REPO} + docker tag riju:$(I) $(DOCKER_REPO):$(I)-$(IMAGE_HASH) + docker push $(DOCKER_REPO):$(I)-$(IMAGE_HASH) docker tag riju:$(I) $(DOCKER_REPO):$(I) docker push $(DOCKER_REPO):$(I) diff --git a/backend/test-runner.js b/backend/test-runner.js index e232a23..868a96b 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -742,7 +742,7 @@ async function main() { await fs.mkdir(`build/test-hashes/lang`, { recursive: true }); await fs.writeFile( `build/test-hashes/lang/${lang}`, - await getTestHash(lang, process.env.RIJU_IMAGE_HASH) + await getTestHash(lang, process.env.RIJU_LANG_IMAGE_HASH) ); } process.exit(failed.size > 0 ? 1 : 0); diff --git a/tools/depgraph.js b/tools/depgraph.js index 65305cd..807ee5b 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -222,7 +222,7 @@ async function getLanguageTestArtifact({ lang }) { return s3TestHashes[lang] || null; }, getDesiredHash: async (dependencyHashes) => { - return await getTestHash(lang, dependencyHashes["image:runtime"]); + return await getTestHash(lang, dependencyHashes[`image:lang-${lang}`]); }, buildLocally: async () => { await runCommand(`make shell I=runtime CMD="make test L=${lang}"`); From 7bb2582c6df59eb4295485b2b5f6ceb8ed5b78ea Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 03:08:34 +0000 Subject: [PATCH 084/104] Misc language fixes --- langs/qsharp.yaml | 13 ++++++++----- langs/reasonml.yaml | 4 +--- langs/red.yaml | 2 +- tools/generate-build-script.js | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/langs/qsharp.yaml b/langs/qsharp.yaml index 14c6224..98f7a42 100644 --- a/langs/qsharp.yaml +++ b/langs/qsharp.yaml @@ -23,16 +23,18 @@ install: # # We could optimize further but I don't feel like it right now. manual: | - install -d "${pkg}/opt/qsharp/skel" + install -d "${pkg}/opt/qsharp/skel-home" + install -d "${pkg}/opt/qsharp/skel-src" dotnet new -i Microsoft.Quantum.ProjectTemplates dotnet new console -lang Q# -o main dotnet run --project main shopt -s dotglob - cp -R * "${HOME}/.dotnet" "${HOME}/.nuget" "${pkg}/opt/qsharp/skel/" - rm "${pkg}/opt/qsharp/skel/main/Program.qs" - chmod -R a=u,go-w "${pkg}/opt/qsharp/skel" + cp -R main "${pkg}/opt/qsharp/skel-src/" + cp -R "${HOME}/.dotnet" "${HOME}/.nuget" "${pkg}/opt/qsharp/skel-home/" + rm "${pkg}/opt/qsharp/skel-src/main/Program.qs" + chmod -R a=u,go-w "${pkg}/opt/qsharp" manualInstall: | wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" sudo --preserve-env=DEBIAN_FRONTEND apt-get update @@ -40,7 +42,8 @@ install: setup: | shopt -s dotglob - cp -R /opt/qsharp/skel/* ./ + cp -R /opt/qsharp/skel-src/* ./ + cp -R /opt/qsharp/skel-home/* "${HOME}/" main: "main/Main.qs" template: | diff --git a/langs/reasonml.yaml b/langs/reasonml.yaml index 12662e9..7b2776d 100644 --- a/langs/reasonml.yaml +++ b/langs/reasonml.yaml @@ -9,8 +9,6 @@ name: "ReasonML" install: prepare: - apt: - - yarn npm: - bs-platform npm: @@ -31,7 +29,7 @@ install: pushd "${pkg}/opt/reasonml/skel" bsb -init . cat bsconfig.json | jq '.name = "riju-project"' | sponge bsconfig.json - yarn install + npm install popd main: "main.re" diff --git a/langs/red.yaml b/langs/red.yaml index 86f82a0..8c69509 100644 --- a/langs/red.yaml +++ b/langs/red.yaml @@ -17,7 +17,7 @@ install: cp -R "$HOME/.red" "${pkg}/opt/red/skel/" setup: | - shopt -s dotglob; cp -R /opt/red/skel/* ./ + shopt -s dotglob; cp -R /opt/red/skel/* "${HOME}/" # https://github.com/red/red/issues/543#issuecomment-25404212 repl: | diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index d594090..cd547ad 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -419,7 +419,7 @@ EOF`); parts.unshift(`ubuntu_name="$(lsb_release -cs)"`); } if (parts.join("\n\n").includes("ubuntu_ver")) { - parts.unshift(`ubuntu_name="$(lsb_release -rs)"`); + parts.unshift(`ubuntu_ver="$(lsb_release -rs)"`); } parts.unshift(`\ #!/usr/bin/env bash From ad9bd56bb89cc8ae2e942aeda5447d79d764937d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 03:08:55 +0000 Subject: [PATCH 085/104] Supervisor script core functionality --- supervisor/go.mod | 10 ++ supervisor/go.sum | 41 +++++ supervisor/src/main.go | 355 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 388 insertions(+), 18 deletions(-) create mode 100644 supervisor/go.sum diff --git a/supervisor/go.mod b/supervisor/go.mod index e766e88..116938f 100644 --- a/supervisor/go.mod +++ b/supervisor/go.mod @@ -1,3 +1,13 @@ module github.com/raxod502/riju/supervisor go 1.16 + +require ( + github.com/aws/aws-sdk-go-v2 v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/config v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.11.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.5.0 // indirect + github.com/caarlos0/env/v6 v6.6.2 // indirect + github.com/google/uuid v1.2.0 // indirect +) diff --git a/supervisor/go.sum b/supervisor/go.sum new file mode 100644 index 0000000..ed79512 --- /dev/null +++ b/supervisor/go.sum @@ -0,0 +1,41 @@ +github.com/aws/aws-sdk-go-v2 v1.7.0 h1:UYGnoIPIzed+ycmgw8Snb/0HK+KlMD+SndLTneG8ncE= +github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= +github.com/aws/aws-sdk-go-v2/config v1.4.1 h1:PcGp9Kf+1dHJmP3EIDZJmAmWfGABFTU0obuvYQNzWH8= +github.com/aws/aws-sdk-go-v2/config v1.4.1/go.mod h1:HCDWZ/oeY59TPtXslxlbkCqLQBsVu6b09kiG43tdP+I= +github.com/aws/aws-sdk-go-v2/credentials v1.3.0 h1:vXxTINCsHn6LKhR043jwSLd6CsL7KOEU7b1woMr1K1A= +github.com/aws/aws-sdk-go-v2/credentials v1.3.0/go.mod h1:tOcv+qDZ0O+6Jk2beMl5JnZX6N0H7O8fw9UsD3bP7GI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.2.0 h1:ucExzYCoAiL9GpKOsKkQLsa43wTT23tcdP4cDTSbZqY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.2.0/go.mod h1:XvzoGzuS0kKPzCQtJCC22Xh/mMgVAzfGo/0V+mk/Cu0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.1 h1:ag1MjvYmE8hnvl2/3LYOog9GZxcguqR6z1ewCUJQ9rE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.1/go.mod h1:WXrj1wxGcYFfQ6H4xqsbVziISWQT55SlpX8B5+EqLOw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0 h1:DJq/vXXF+LAFaa/kQX9C6arlf4xX4uaaqGWIyAKOCpM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0/go.mod h1:qGQ/9IfkZonRNSNLE99/yBJ7EPA/h8jlWEqtJCcaj+Q= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.0 h1:wfI4yrOCMAGdHaEreQ65ycSmPLVc2Q82O+r7ZxYTynA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.0/go.mod h1:2Kc2Pybp1Hr2ZCCOz78mWnNSZYEKKBQgNcizVGk9sko= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.0 h1:g2npzssI/6XsoQaPYCxliMFeC5iNKKvO0aC+/wWOE0A= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.0/go.mod h1:a7XLWNKuVgOxjssEF019IiHPv35k8KHBaWv/wJAfi2A= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.0 h1:6KmDU3XCGTcZlWPtP/gh7wYErrovnIxjX7um8iiuVsU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.0/go.mod h1:541bxEA+Z8quwit9ZT7uxv/l9xRz85/HS41l9OxOQdY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.11.0 h1:FuKlyrDBZBk0RFxjqFPtx9y/KDsxTa3MoFVUgIW9w3Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.11.0/go.mod h1:zJe8mEFDS2F04nO0pKVBPfArAv2ycC6wt3ILvrV4SQw= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.0 h1:DMi9w+TpUam7eJ8ksL7svfzpqpqem2MkDAJKW8+I2/k= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.0/go.mod h1:qWR+TUuvfji9udM79e4CPe87C5+SjMEb2TFXkZaI0Vc= +github.com/aws/aws-sdk-go-v2/service/sts v1.5.0 h1:Y1K9dHE2CYOWOvaJSIITq4mJfLX43iziThTvqs5FqOg= +github.com/aws/aws-sdk-go-v2/service/sts v1.5.0/go.mod h1:HjDKUmissf6Mlut+WzG2r35r6LeTKmLEDJ6p9NryzLg= +github.com/aws/smithy-go v1.5.0 h1:2grDq7LxZlo8BZUDeqRfQnQWLZpInmh2TLPPkJku3YM= +github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/caarlos0/env/v6 v6.6.2 h1:BypLXDWQTA32rS4UM7pBz+/0BOuvs6C7LSeQAxMwyvI= +github.com/caarlos0/env/v6 v6.6.2/go.mod h1:P0BVSgU9zfkxfSpFUs6KsO3uWR4k3Ac0P66ibAGTybM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 433e4ae..85f1673 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -1,75 +1,394 @@ package main import ( + "context" + "encoding/json" + "errors" "fmt" + "io" "log" "net/http" "net/http/httputil" "net/url" + "os" + "os/exec" + "regexp" "strings" "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/caarlos0/env/v6" + uuidlib "github.com/google/uuid" ) +const bluePort = 6229 +const greenPort = 6230 + +const blueName = "riju-app-blue" +const greenName = "riju-app-green" + +type deploymentConfig struct { + LangImageTags map[string]string `json:"langImageTags"` + AppImageTag string `json:"appImageTag"` +} + +type supervisorConfig struct { + AccessToken string `env:"SUPERVISOR_ACCESS_TOKEN,notEmpty"` + S3Bucket string `env:"S3_BUCKET,notEmpty"` +} + +type reloadJob struct { + status string + active bool + failed bool +} + type supervisor struct { - proxyHandler http.Handler + config supervisorConfig + + blueProxyHandler http.Handler + greenProxyHandler http.Handler + isGreen bool // blue-green deployment + + awsAccountNumber string + awsRegion string + s3 *s3.Client reloadLock sync.Mutex reloadInProgress bool reloadNeeded bool + reloadUUID string + reloadNextUUID string + reloadJobs map[string]*reloadJob +} + +func (sv *supervisor) status(status string) { + sv.reloadLock.Lock() + sv.reloadJobs[sv.reloadUUID].status = status + sv.reloadLock.Unlock() + log.Println("active: " + status) } func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/api/supervisor") { if r.URL.Path == "/api/supervisor/v1/reload" { - resp := "" + if r.Method != http.MethodPost { + http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) + return + } + uuid := "" sv.reloadLock.Lock() if !sv.reloadInProgress { sv.reloadInProgress = true - go sv.reload() - resp = "Triggered reload." + sv.reloadUUID = uuidlib.New().String() + uuid = sv.reloadUUID + go sv.reloadWithScheduling() } else { + if sv.reloadInProgress { + uuid = sv.reloadNextUUID + } else { + sv.reloadNextUUID = uuidlib.New().String() + uuid = sv.reloadNextUUID + } sv.reloadNeeded = true - resp = "Reload already in progress, scheduling another one." } sv.reloadLock.Unlock() - fmt.Fprintln(w, resp) + fmt.Fprintln(w, uuid) + return + } + if r.URL.Path == "/api/supervisor/v1/reload/status" { + if r.Method != http.MethodGet { + http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) + return + } + uuids := r.URL.Query()["uuid"] + if len(uuids) == 0 { + http.Error( + w, + "400 missing uuid query parameter", + http.StatusBadRequest, + ) + return + } + if len(uuids) > 1 { + http.Error( + w, + "400 more than one uuid query parameter", + http.StatusBadRequest, + ) + return + } + uuid := uuids[0] + sv.reloadLock.Lock() + job := sv.reloadJobs[uuid] + if job == nil { + if uuid == sv.reloadUUID || uuid == sv.reloadNextUUID { + fmt.Fprintln(w, "queued") + } else { + http.Error(w, "404 no such job", http.StatusNotFound) + } + } else if job.active { + fmt.Fprintln(w, "active: " + job.status) + } else if job.failed { + fmt.Fprintln(w, "failed: " + job.status) + } else { + fmt.Fprintln(w, "succeeded: " + job.status) + } + sv.reloadLock.Unlock() return } http.NotFound(w, r) return } - sv.proxyHandler.ServeHTTP(w, r) + if sv.isGreen { + sv.greenProxyHandler.ServeHTTP(w, r) + } else { + sv.blueProxyHandler.ServeHTTP(w, r) + } return } func (sv *supervisor) reloadWithScheduling() { - err := sv.reload() - if err != nil { - log.Printf("failed to reload: %s\n", err.Error()) - } else { - log.Println("successfully reloaded") - } sv.reloadLock.Lock() + sv.reloadJobs[sv.reloadUUID] = &reloadJob{ + status: "initializing", + active: true, + failed: false, + } + sv.reloadLock.Unlock() + err := sv.reload() + sv.reloadLock.Lock() + sv.reloadJobs[sv.reloadUUID].active = false + if err != nil { + log.Println("failed: " + err.Error()) + sv.reloadJobs[sv.reloadUUID].failed = true + sv.reloadJobs[sv.reloadUUID].status = err.Error() + } else { + log.Println("succeeded") + } sv.reloadInProgress = false + sv.reloadUUID = "" if sv.reloadNeeded { + sv.reloadNeeded = false sv.reloadInProgress = true + sv.reloadUUID = sv.reloadNextUUID + sv.reloadNextUUID = "" go sv.reloadWithScheduling() } sv.reloadLock.Unlock() } +var rijuImageRegexp = regexp.MustCompile(`(?:^|/)riju:([^<>]+)$`) + func (sv *supervisor) reload() error { - fmt.Println("starting reload") + sv.status("downloading deployment config from S3") + dl := s3manager.NewDownloader(sv.s3) + buf := s3manager.NewWriteAtBuffer([]byte{}) + if _, err := dl.Download(context.Background(), buf, &s3.GetObjectInput{ + Bucket: &sv.config.S3Bucket, + Key: aws.String("config.json"), + }); err != nil { + return err + } + deployCfg := deploymentConfig{} + if err := json.Unmarshal(buf.Bytes(), &deployCfg); err != nil { + return err + } + sv.status("listing locally available images") + dockerImageLs := exec.Command( + "docker", "image", "ls", "--format", + "{{ .Repository }}:{{ .Tag }}", + ) + out, err := dockerImageLs.Output() + if err != nil { + return err + } + existingTags := map[string]bool{} + for _, line := range strings.Split(string(out), "\n") { + if match := rijuImageRegexp.FindStringSubmatch(line); match != nil { + tag := match[1] + existingTags[tag] = true + } + } + neededTags := []string{} + for _, tag := range deployCfg.LangImageTags { + neededTags = append(neededTags, tag) + } + neededTags = append(neededTags, deployCfg.AppImageTag) + for _, tag := range neededTags { + if !existingTags[tag] { + sv.status("pulling image for " + tag) + fullImage := fmt.Sprintf( + "%s.dkr.ecr.%s.amazonaws.com/riju:%s", + sv.awsAccountNumber, + sv.awsRegion, + tag, + ) + dockerPull := exec.Command("docker", "pull", fullImage) + if err := dockerPull.Run(); err != nil { + return err + } + dockerTag := exec.Command( + "docker", "tag", fullImage, + fmt.Sprintf("riju:%s", tag), + ) + if err := dockerTag.Run(); err != nil { + return err + } + } + } + deployCfgStr, err := json.Marshal(&deployCfg) + if err != nil { + return err + } + var port int + var name string + if sv.isGreen { + port = bluePort + name = blueName + } else { + port = greenPort + name = greenName + } + sv.status("starting container " + name) + dockerRun := exec.Command( + "docker", "run", "-d", + "-v", "/var/run/riju:/var/run/riju", + "-v", "/var/run/docker.sock:/var/run/docker.sock", + "-p", fmt.Sprintf("%s:6119", port), + "-e", "RIJU_DEPLOY_CONFIG", + "--name", name, + fmt.Sprintf("riju:%s", deployCfg.AppImageTag), + ) + dockerRun.Env = append(os.Environ(), fmt.Sprintf("RIJU_DEPLOY_CONFIG=%s", deployCfgStr)) + if err := dockerRun.Run(); err != nil { + return err + } + sv.status("waiting for container to start up") + time.Sleep(5) + sv.status("checking that container is healthy") + resp, err := http.Get(fmt.Sprintf("http://localhost:%d", port)) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if !strings.Contains(string(body), "python") { + return errors.New("container did not appear to be healthy") + } + sv.isGreen = !sv.isGreen + sv.status("reload complete") return nil } +var rijuContainerRegexp = regexp.MustCompile(`^([^:]+):(.+)$`) + func main() { - url, err := url.Parse("http://localhost:6119") - if err != nil { - log.Fatal(err) + supervisorCfg := supervisorConfig{} + if err := env.Parse(&supervisorCfg); err != nil { + log.Fatalln(err) } + + blueUrl, err := url.Parse(fmt.Sprintf("http://localhost:%d", bluePort)) + if err != nil { + log.Fatalln(err) + } + greenUrl, err := url.Parse(fmt.Sprintf("http://localhost:%d", greenPort)) + if err != nil { + log.Fatalln(err) + } + + awsCfg, err := awsConfig.LoadDefaultConfig(context.Background()) + if err != nil { + log.Fatalln(err) + } + + stsClient := sts.New(sts.Options{ + Region: awsCfg.Region, + }) + ident, err := stsClient.GetCallerIdentity(context.Background(), &sts.GetCallerIdentityInput{}) + if err != nil { + log.Fatalln(err) + } + + dockerContainerLs := exec.Command( + "docker", "container", "ls", + "--format", "{{ .Names }}:{{ .CreatedAt }}", + ) + out, err := dockerContainerLs.Output() + if err != nil { + log.Fatalln(err) + } + + var blueRunningSince *time.Time + var greenRunningSince *time.Time + for _, line := range strings.Split(string(out), "\n") { + if match := rijuContainerRegexp.FindStringSubmatch(line); match != nil { + name := match[1] + created, err := time.Parse( + "2006-01-02 15:04:05 -0070 MST", + match[2], + ) + if err != nil { + continue + } + if name == blueName { + blueRunningSince = &created + continue + } + if name == greenName { + greenRunningSince = &created + continue + } + } + } + + var isGreen bool + if blueRunningSince == nil && greenRunningSince == nil { + log.Println("did not detect any existing containers") + isGreen = false + } else if blueRunningSince != nil && greenRunningSince == nil { + log.Println("detected existing blue container") + isGreen = false + } else if greenRunningSince != nil && blueRunningSince == nil { + log.Println("detected existing green container") + isGreen = true + } else { + log.Println("detected existing blue and green containers") + isGreen = greenRunningSince.Before(*blueRunningSince) + var color string + var name string + if isGreen { + color = "blue" + name = blueName + } else { + color = "green" + name = greenName + } + log.Printf("stopping %s container as it is newer\n", color) + dockerStop := exec.Command("docker", "stop", name) + if err := dockerStop.Run(); err != nil { + log.Fatalln(err) + } + } + sv := &supervisor{ - proxyHandler: httputil.NewSingleHostReverseProxy(url), + config: supervisorCfg, + blueProxyHandler: httputil.NewSingleHostReverseProxy(blueUrl), + greenProxyHandler: httputil.NewSingleHostReverseProxy(greenUrl), + isGreen: isGreen, + s3: s3.NewFromConfig(awsCfg), + awsRegion: awsCfg.Region, + awsAccountNumber: *ident.Account, + reloadJobs: map[string]*reloadJob{}, } log.Println("listening on http://0.0.0.0:80") log.Fatalln(http.ListenAndServe("0.0.0.0:80", sv)) From 130fb8b34c7ff1dfbd39163fbe754cd51851cdd5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 03:09:08 +0000 Subject: [PATCH 086/104] Fix PATH bug --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3e5a59e..4138d80 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL := bash .SHELLFLAGS := -o pipefail -euc -export PATH := /src/bin:$(PATH) +export PATH := $(PWD)/bin:$(PATH) -include .env export From 2062bfecdbb1551cc7bbc6646cd429b784e48ba5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 03:29:02 +0000 Subject: [PATCH 087/104] First attempt at new AMI build --- Makefile | 7 ++++- packer/config.json | 39 +++------------------------ packer/provision.bash | 30 +++------------------ packer/riju | 44 ------------------------------- packer/riju-deploy | 35 ------------------------ packer/riju-install-certbot-hooks | 18 ------------- packer/riju.service | 2 +- packer/validate.bash | 7 ----- supervisor/src/main.go | 1 + tools/packer-build.bash | 6 +++++ 10 files changed, 22 insertions(+), 167 deletions(-) delete mode 100755 packer/riju delete mode 100755 packer/riju-deploy delete mode 100755 packer/riju-install-certbot-hooks delete mode 100755 packer/validate.bash create mode 100755 tools/packer-build.bash diff --git a/Makefile b/Makefile index 4138d80..4e4627b 100644 --- a/Makefile +++ b/Makefile @@ -230,6 +230,11 @@ config: # Generate deployment config file deploy: # Upload deployment config to S3 aws s3 cp $(BUILD)/config.json $(S3_CONFIG) +### Infrastructure + +packer: supervisor # Build and publish a new AMI + tools/packer-build.bash + ### Miscellaneous ## Run this every time you update .gitignore or .dockerignore.in. @@ -243,7 +248,7 @@ dockerignore: # Update .dockerignore from .gitignore and .dockerignore.in ## opposed to through the Makefile. env: # Run shell with .env file loaded and $PATH fixed - exec bash --rcfile <(cat ~/.bashrc - <<< 'PS1="[.env] $$PS1"') + exec bash tmux: # Start or attach to tmux session MAKELEVEL= tmux attach || MAKELEVEL= tmux new-session -s tmux diff --git a/packer/config.json b/packer/config.json index ecfd3a7..e4f6f8e 100644 --- a/packer/config.json +++ b/packer/config.json @@ -1,8 +1,6 @@ { "variables": { - "admin_password": "{{env `ADMIN_PASSWORD`}}", - "admin_ssh_public_key_file": "{{env `ADMIN_SSH_PUBLIC_KEY_FILE`}}", - "deploy_ssh_public_key_file": "{{env `DEPLOY_SSH_PUBLIC_KEY_FILE`}}" + "admin_password": "{{env `ADMIN_PASSWORD`}}" }, "builders": [ { @@ -11,7 +9,7 @@ "filters": { "virtualization-type": "hvm", "root-device-type": "ebs", - "name": "ubuntu/images/hvm-ssd/ubuntu-groovy-20.10-amd64-server-*" + "name": "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*" }, "owners": ["099720109477"], "most_recent": true @@ -22,20 +20,6 @@ } ], "provisioners": [ - { - "type": "shell", - "script": "validate.bash", - "environment_vars": [ - "ADMIN_PASSWORD={{user `admin_password`}}", - "ADMIN_SSH_PUBLIC_KEY_FILE={{user `admin_ssh_public_key_file`}}", - "DEPLOY_SSH_PUBLIC_KEY_FILE={{user `deploy_ssh_public_key_file`}}" - ] - }, - { - "type": "file", - "source": "riju", - "destination": "/tmp/riju" - }, { "type": "file", "source": "riju-init-volume", @@ -43,29 +27,14 @@ }, { "type": "file", - "source": "riju-deploy", - "destination": "/tmp/riju-deploy" - }, - { - "type": "file", - "source": "riju-install-certbot-hooks", - "destination": "/tmp/riju-install-certbot-hooks" + "source": "../supervisor/out/riju-supervisor", + "destination": "/tmp/riju-supervisor" }, { "type": "file", "source": "riju.service", "destination": "/tmp/riju.service" }, - { - "type": "file", - "source": "{{user `admin_ssh_public_key_file`}}", - "destination": "/tmp/id_admin.pub" - }, - { - "type": "file", - "source": "{{user `deploy_ssh_public_key_file`}}", - "destination": "/tmp/id_deploy.pub" - }, { "type": "shell", "script": "provision.bash", diff --git a/packer/provision.bash b/packer/provision.bash index 303041f..f2f0f73 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -2,6 +2,8 @@ set -euo pipefail +: ${ADMIN_PASSWORD} + mkdir /tmp/riju-work pushd /tmp/riju-work @@ -27,40 +29,16 @@ wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip unzip -q awscli.zip sudo ./aws/install -sudo chown root:root /tmp/riju /tmp/riju-deploy /tmp/riju.service -sudo mv /tmp/riju /tmp/riju-deploy /tmp/riju-init-volume /tmp/riju-install-certbot-hooks /usr/local/bin/ +sudo chown root:root /tmp/riju-init-volume /tmp/riju-supervisor /tmp/riju.service +sudo mv /tmp/riju-init-volume /tmp/riju-supervisor /usr/local/bin/ sudo mv /tmp/riju.service /etc/systemd/system/ -for user in admin deploy; do - if ! grep -vq "PRIVATE KEY" "/tmp/id_${user}.pub"; then - echo "${user} public key was set to a private key, aborting" >&2 - exit 1 - fi - - IFS=" " read contents < "/tmp/id_${user}.pub" - echo "${contents}" > "/tmp/id_${user}.pub" -done - sudo sed -Ei 's/^#?PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config sudo sed -Ei 's/^#?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config sudo sed -Ei 's/^#?PermitEmptyPasswords .*/PermitEmptyPasswords no/' /etc/ssh/sshd_config sudo passwd -l root sudo useradd admin -g admin -G sudo -s /usr/bin/bash -p "$(echo "${ADMIN_PASSWORD}" | mkpasswd -s)" -m -sudo useradd deploy -s /usr/bin/bash -p "!" -m - -for user in admin deploy; do - sudo runuser -u "${user}" -- mkdir -p "/home/${user}/.ssh" - sudo mv "/tmp/id_${user}.pub" "/home/${user}/.ssh/authorized_keys" - sudo chown -R "${user}:${user}" "/home/${user}/.ssh" - sudo chmod -R go-rwx "/home/${user}/.ssh" -done - -sudo runuser -u deploy -- sed -i 's/^/command="sudo riju-deploy ${SSH_ORIGINAL_COMMAND}",restrict /' /home/deploy/.ssh/authorized_keys - -sudo tee /etc/sudoers.d/riju >/dev/null <<"EOF" -deploy ALL=(root) NOPASSWD: /usr/local/bin/riju-deploy -EOF sudo hostnamectl set-hostname riju diff --git a/packer/riju b/packer/riju deleted file mode 100755 index 552102c..0000000 --- a/packer/riju +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -domain="$(ls /etc/letsencrypt/live | grep -v README | head -n1)" || true - -if [[ -n "${DISABLE_TLS:-}" ]]; then - echo "Disabling TLS due to DISABLE_TLS=${DISABLE_TLS}" >&2 -elif [[ -z "${domain}" ]]; then - echo "No certs installed in /etc/letsencrypt/live, disabling TLS" >&2 -else - echo "Detected cert for domain: ${domain}, enabling TLS" >&2 - export TLS=1 - TLS_PRIVATE_KEY="$(base64 "/etc/letsencrypt/live/${domain}/privkey.pem")" - TLS_CERTIFICATE="$(base64 "/etc/letsencrypt/live/${domain}/fullchain.pem")" - export TLS_PRIVATE_KEY TLS_CERTIFICATE - if [[ "${domain}" == riju.codes ]]; then - echo "Domain is riju.codes, enabling analytics" >&2 - export ANALYTICS=1 - else - echo "Domain is not riju.codes, disabling analytics" >&2 - fi -fi - -if [[ -n "${DETACH:-}" ]]; then - extra_args="-d" -elif [[ -t 1 ]]; then - extra_args="-it" -else - extra_args= -fi - -port_args="${PORT_MAPPING:--p 0.0.0.0:80:6119 -p 0.0.0.0:443:6120}" -image_name="${IMAGE_NAME:-riju:app}" -container_name="${CONTAINER_NAME:-riju-prod}" - -if docker container inspect ${container_name} &>/dev/null; then - docker stop ${container_name} -fi - -docker run --rm ${port_args} ${extra_args} \ - -e TLS -e TLS_PRIVATE_KEY -e TLS_CERTIFICATE -e ANALYTICS \ - -h riju --name "${container_name}" \ - "${image_name}" diff --git a/packer/riju-deploy b/packer/riju-deploy deleted file mode 100755 index f978f66..0000000 --- a/packer/riju-deploy +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if (( $# != 1 )); then - echo "usage: ssh deploy@riju IMAGE" >&2 - exit 1 -fi - -image="$1" - -riju-init-volume - -echo "Pull image to be deployed..." -docker pull "${image}" - -echo "Start new image in test container..." >&2 -CONTAINER_NAME=riju-test IMAGE_NAME="${image}" DETACH=1 \ - PORT_MAPPING="-p 127.0.0.1:6119:6119" DISABLE_TLS=0 riju - -echo "Wait for web server to come up..." >&2 -sleep 5 - -echo "Test web server health..." >&2 -output="$(curl -fsSL http://localhost:6119)" -head -n15 <<< "${output}" - -echo "Tear down test container..." >&2 -docker stop riju-test - -echo "Retag production image..." >&2 -docker tag "${image}" riju:app - -echo "Restart production server..." >&2 -systemctl restart riju diff --git a/packer/riju-install-certbot-hooks b/packer/riju-install-certbot-hooks deleted file mode 100755 index d83cd8b..0000000 --- a/packer/riju-install-certbot-hooks +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -sudo tee /etc/letsencrypt/renewal-hooks/pre/riju >/dev/null <<"EOF" -#!/usr/bin/env bash -set -euo pipefail -systemctl stop riju -EOF - -sudo tee /etc/letsencrypt/renewal-hooks/post/riju >/dev/null <<"EOF" -#!/usr/bin/env bash -set -euo pipefail -systemctl start riju -EOF - -sudo chmod +x /etc/letsencrypt/renewal-hooks/pre/riju -sudo chmod +x /etc/letsencrypt/renewal-hooks/post/riju diff --git a/packer/riju.service b/packer/riju.service index 6ea3ac0..10a121c 100644 --- a/packer/riju.service +++ b/packer/riju.service @@ -5,7 +5,7 @@ After=docker.service [Service] Type=exec -ExecStart=riju +ExecStart=riju-supervisor Restart=always [Install] diff --git a/packer/validate.bash b/packer/validate.bash deleted file mode 100755 index ab4f836..0000000 --- a/packer/validate.bash +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -: ${ADMIN_PASSWORD} -: ${ADMIN_SSH_PUBLIC_KEY_FILE} -: ${DEPLOY_SSH_PUBLIC_KEY_FILE} diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 85f1673..5762c1e 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -262,6 +262,7 @@ func (sv *supervisor) reload() error { "-v", "/var/run/docker.sock:/var/run/docker.sock", "-p", fmt.Sprintf("%s:6119", port), "-e", "RIJU_DEPLOY_CONFIG", + "-e", "ANALYTICS=1", "--name", name, fmt.Sprintf("riju:%s", deployCfg.AppImageTag), ) diff --git a/tools/packer-build.bash b/tools/packer-build.bash new file mode 100755 index 0000000..f20726b --- /dev/null +++ b/tools/packer-build.bash @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd packer +packer build config.json From be7403e3673f9f9060ba0ac44fde1e4242a097b5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 04:24:47 +0000 Subject: [PATCH 088/104] Misc Packer improvements --- packer/config.json | 44 -------------------------- packer/config.pkr.hcl | 69 +++++++++++++++++++++++++++++++++++++++++ packer/provision.bash | 7 ++++- packer/riju.service | 3 ++ tools/packer-build.bash | 9 +++++- 5 files changed, 86 insertions(+), 46 deletions(-) delete mode 100644 packer/config.json create mode 100644 packer/config.pkr.hcl diff --git a/packer/config.json b/packer/config.json deleted file mode 100644 index e4f6f8e..0000000 --- a/packer/config.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "variables": { - "admin_password": "{{env `ADMIN_PASSWORD`}}" - }, - "builders": [ - { - "type": "amazon-ebs", - "source_ami_filter": { - "filters": { - "virtualization-type": "hvm", - "root-device-type": "ebs", - "name": "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*" - }, - "owners": ["099720109477"], - "most_recent": true - }, - "instance_type": "t3.micro", - "ssh_username": "ubuntu", - "ami_name": "riju-{{timestamp}}" - } - ], - "provisioners": [ - { - "type": "file", - "source": "riju-init-volume", - "destination": "/tmp/riju-init-volume" - }, - { - "type": "file", - "source": "../supervisor/out/riju-supervisor", - "destination": "/tmp/riju-supervisor" - }, - { - "type": "file", - "source": "riju.service", - "destination": "/tmp/riju.service" - }, - { - "type": "shell", - "script": "provision.bash", - "environment_vars": ["ADMIN_PASSWORD={{user `admin_password`}}"] - } - ] -} diff --git a/packer/config.pkr.hcl b/packer/config.pkr.hcl new file mode 100644 index 0000000..49d213e --- /dev/null +++ b/packer/config.pkr.hcl @@ -0,0 +1,69 @@ +variable "admin_password" { + type = string + default = "${env("ADMIN_PASSWORD")}" +} + +variable "aws_region" { + type = string + default = "${env("AWS_REGION")}" +} + +variable "s3_bucket" { + type = string + default = "${env("S3_BUCKET")}" +} + +variable "supervisor_access_token" { + type = string + default = "${env("SUPERVISOR_ACCESS_TOKEN")}" +} + +data "amazon-ami" "ubuntu" { + filters = { + name = "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] +} + +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") +} + +source "amazon-ebs" "ubuntu" { + ami_name = "riju-${local.timestamp}" + instance_type = "t3.micro" + source_ami = "${data.amazon-ami.ubuntu.id}" + ssh_username = "ubuntu" +} + +build { + sources = ["source.amazon-ebs.ubuntu"] + + provisioner "file" { + destination = "/tmp/riju-init-volume" + source = "riju-init-volume" + } + + provisioner "file" { + destination = "/tmp/riju-supervisor" + source = "../supervisor/out/riju-supervisor" + } + + provisioner "file" { + destination = "/tmp/riju.service" + source = "riju.service" + } + + provisioner "shell" { + environment_vars = [ + "ADMIN_PASSWORD=${var.admin_password}", + "AWS_REGION=${var.aws_region}", + "S3_BUCKET=${var.s3_bucket}", + "SUPERVISOR_ACCESS_TOKEN=${var.supervisor_access_token}", + ] + script = "provision.bash" + } +} diff --git a/packer/provision.bash b/packer/provision.bash index f2f0f73..046b79e 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -3,6 +3,8 @@ set -euo pipefail : ${ADMIN_PASSWORD} +: ${S3_BUCKET} +: ${SUPERVISOR_ACCESS_TOKEN} mkdir /tmp/riju-work pushd /tmp/riju-work @@ -21,7 +23,7 @@ ubuntu_name="$(lsb_release -cs)" sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null <&2 "no default AWS region specified, and AWS_REGION unset" + exit 1 +fi + cd packer -packer build config.json +packer build config.pkr.hcl From 44813bb6d5fc0de6810fea1769a18028ce31ed43 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 04:25:02 +0000 Subject: [PATCH 089/104] Split up Terraform --- tf/acm.tf | 13 ++ tf/alb.tf | 47 +++++++ tf/asg.tf | 115 +++++++++++++++++ tf/ecr.tf | 4 + tf/iam.tf | 72 +++++++++++ tf/infra.tf | 336 -------------------------------------------------- tf/main.tf | 51 ++++++++ tf/outputs.tf | 12 ++ tf/s3.tf | 17 +++ 9 files changed, 331 insertions(+), 336 deletions(-) create mode 100644 tf/acm.tf create mode 100644 tf/alb.tf create mode 100644 tf/asg.tf create mode 100644 tf/ecr.tf create mode 100644 tf/iam.tf delete mode 100644 tf/infra.tf create mode 100644 tf/main.tf create mode 100644 tf/outputs.tf create mode 100644 tf/s3.tf diff --git a/tf/acm.tf b/tf/acm.tf new file mode 100644 index 0000000..ef4e7fa --- /dev/null +++ b/tf/acm.tf @@ -0,0 +1,13 @@ +resource "aws_acm_certificate" "riju" { + domain_name = "riju.codes" + subject_alternative_names = ["*.riju.codes"] + validation_method = "DNS" + + tags = { + Name = "Riju server" + } +} + +resource "aws_acm_certificate_validation" "riju" { + certificate_arn = aws_acm_certificate.riju.arn +} diff --git a/tf/alb.tf b/tf/alb.tf new file mode 100644 index 0000000..b8a271b --- /dev/null +++ b/tf/alb.tf @@ -0,0 +1,47 @@ +resource "aws_security_group" "alb" { + name = "riju-alb" + description = "Security group for Riju application load balancer" + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTPS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_lb" "server" { + name = "riju-server" + security_groups = [aws_security_group.alb.id] + subnets = data.aws_subnet_ids.default.ids +} + +resource "aws_lb_target_group" "server_http" { + name = "riju-server-http" + port = 80 + protocol = "HTTP" + vpc_id = data.aws_vpc.default.id +} + +resource "aws_lb_target_group" "server_https" { + name = "riju-server-https" + port = 443 + protocol = "HTTPS" + vpc_id = data.aws_vpc.default.id +} diff --git a/tf/asg.tf b/tf/asg.tf new file mode 100644 index 0000000..3e98b93 --- /dev/null +++ b/tf/asg.tf @@ -0,0 +1,115 @@ +data "aws_ami" "server" { + count = local.ami_available ? 1 : 0 + + owners = ["self"] + + filter { + name = "name" + values = [data.external.env.result.AMI_NAME] + } +} + +resource "aws_security_group" "server" { + name = "riju-server" + description = "Security group for Riju server" + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTPS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_launch_template" "server" { + count = local.ami_available ? 1 : 0 + + name = "riju-server" + image_id = data.aws_ami.server[0].id + instance_type = "t3.small" + security_group_names = [aws_security_group.server.name] + + update_default_version = true + + block_device_mappings { + device_name = "/dev/sdh" + ebs { + volume_type = "gp3" + volume_size = 125 + } + } + + tags = { + Name = "Riju server" + } + + tag_specifications { + resource_type = "instance" + tags = { + Name = "Riju server" + } + } +} + +resource "aws_autoscaling_group" "server" { + count = local.ami_available ? 1 : 0 + + name = "riju-server" + + availability_zones = [ + for subnet in data.aws_subnet.default : subnet.availability_zone + ] + desired_capacity = 1 + min_size = 1 + max_size = 3 + + launch_template { + id = aws_launch_template.server[0].id + } + + target_group_arns = [ + aws_lb_target_group.server_http.arn, + aws_lb_target_group.server_https.arn, + ] + + tags = concat( + [ + { + key = "Name" + value = "Riju server" + propagate_at_launch = false + } + ], + [ + for key, value in local.tags : { + key = key, + value = value, + propagate_at_launch = true, + } + ], + ) +} diff --git a/tf/ecr.tf b/tf/ecr.tf new file mode 100644 index 0000000..9202998 --- /dev/null +++ b/tf/ecr.tf @@ -0,0 +1,4 @@ +resource "aws_ecr_repository" "riju" { + name = "riju" + image_tag_mutability = "IMMUTABLE" +} diff --git a/tf/iam.tf b/tf/iam.tf new file mode 100644 index 0000000..acd8066 --- /dev/null +++ b/tf/iam.tf @@ -0,0 +1,72 @@ +resource "aws_iam_user" "deploy" { + name = "riju-deploy" +} + +resource "aws_iam_access_key" "deploy" { + user = aws_iam_user.deploy.name +} + +data "aws_iam_policy_document" "deploy" { + statement { + actions = [ + "s3:ListBucket", + ] + + resources = [ + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}", + ] + } + + statement { + actions = [ + "s3:*Object", + ] + + resources = [ + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*", + ] + } +} + +resource "aws_iam_policy" "deploy" { + name = "riju-deploy" + description = "Role used by CI to deploy Riju" + policy = data.aws_iam_policy_document.deploy.json +} + +resource "aws_iam_user_policy_attachment" "deploy" { + user = aws_iam_user.deploy.name + policy_arn = aws_iam_policy.deploy.arn +} + +data "aws_iam_policy_document" "riju" { + statement { + principals { + type = "*" + identifiers = ["*"] + } + + actions = [ + "s3:ListBucket", + ] + + resources = [ + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}", + ] + } + + statement { + principals { + type = "*" + identifiers = ["*"] + } + + actions = [ + "s3:GetObject", + ] + + resources = [ + "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*", + ] + } +} diff --git a/tf/infra.tf b/tf/infra.tf deleted file mode 100644 index b76b5ce..0000000 --- a/tf/infra.tf +++ /dev/null @@ -1,336 +0,0 @@ -terraform { - backend "s3" { - key = "state" - region = "us-west-1" - } - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 3.45" - } - null = { - source = "hashicorp/null" - version = "~> 3.1" - } - } -} - -data "external" "env" { - program = ["jq", "-n", "env"] -} - -locals { - tags = { - Terraform = "Managed by Terraform" - BillingCategory = "Riju" - } - - ami_available = lookup(data.external.env.result, "AMI_NAME", "") != "" ? true : false -} - -provider "aws" { - region = "us-west-1" - default_tags { - tags = local.tags - } -} - -data "aws_region" "current" {} - -data "aws_vpc" "default" { - default = true -} - -data "aws_subnet_ids" "default" { - vpc_id = data.aws_vpc.default.id -} - -data "aws_subnet" "default" { - for_each = data.aws_subnet_ids.default.ids - id = each.value -} - -resource "aws_iam_user" "deploy" { - name = "riju-deploy" -} - -resource "aws_iam_access_key" "deploy" { - user = aws_iam_user.deploy.name -} - -data "aws_iam_policy_document" "deploy" { - statement { - actions = [ - "s3:ListBucket", - ] - - resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju.bucket}", - ] - } - - statement { - actions = [ - "s3:*Object", - ] - - resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*", - ] - } -} - -resource "aws_iam_policy" "deploy" { - name = "riju-deploy" - description = "Role used by CI to deploy Riju" - policy = data.aws_iam_policy_document.deploy.json -} - -resource "aws_iam_user_policy_attachment" "deploy" { - user = aws_iam_user.deploy.name - policy_arn = aws_iam_policy.deploy.arn -} - -data "aws_iam_policy_document" "riju" { - statement { - principals { - type = "*" - identifiers = ["*"] - } - - actions = [ - "s3:ListBucket", - ] - - resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju.bucket}", - ] - } - - statement { - principals { - type = "*" - identifiers = ["*"] - } - - actions = [ - "s3:GetObject", - ] - - resources = [ - "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/*", - ] - } -} - -resource "aws_acm_certificate" "riju" { - domain_name = "riju.codes" - subject_alternative_names = ["*.riju.codes"] - validation_method = "DNS" - - tags = { - Name = "Riju server" - } -} - -resource "aws_acm_certificate_validation" "riju" { - certificate_arn = aws_acm_certificate.riju.arn -} - -resource "aws_s3_bucket" "riju" { - bucket = data.external.env.result.S3_BUCKET -} - -resource "aws_s3_bucket_public_access_block" "riju" { - bucket = aws_s3_bucket.riju.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_policy" "riju" { - bucket = aws_s3_bucket.riju.id - policy = data.aws_iam_policy_document.riju.json -} - -resource "aws_ecr_repository" "riju" { - name = "riju" - image_tag_mutability = "IMMUTABLE" -} - -data "aws_ami" "server" { - count = local.ami_available ? 1 : 0 - - owners = ["self"] - - filter { - name = "name" - values = [data.external.env.result.AMI_NAME] - } -} - -resource "aws_security_group" "server" { - name = "riju-server" - description = "Security group for Riju server" - - ingress { - description = "SSH" - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - description = "HTTP" - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - description = "HTTPS" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_security_group" "alb" { - name = "riju-alb" - description = "Security group for Riju application load balancer" - - ingress { - description = "HTTP" - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - description = "HTTPS" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_launch_template" "server" { - count = local.ami_available ? 1 : 0 - - name = "riju-server" - image_id = data.aws_ami.server[0].id - instance_type = "t3.small" - security_group_names = [aws_security_group.server.name] - - block_device_mappings { - device_name = "/dev/sdh" - ebs { - volume_type = "gp3" - volume_size = 125 - } - } - - tags = { - Name = "Riju server" - } - - tag_specifications { - resource_type = "instance" - tags = { - Name = "Riju server" - } - } -} - -resource "aws_autoscaling_group" "server" { - count = local.ami_available ? 1 : 0 - - name = "riju-server" - - availability_zones = [ - for subnet in data.aws_subnet.default : subnet.availability_zone - ] - desired_capacity = 0 - min_size = 0 - max_size = 3 - - launch_template { - id = aws_launch_template.server[0].id - } - - target_group_arns = [ - aws_lb_target_group.server_http.arn, - aws_lb_target_group.server_https.arn, - ] - - tags = concat( - [ - { - key = "Name" - value = "Riju server" - propagate_at_launch = false - } - ], - [ - for key, value in local.tags : { - key = key, - value = value, - propagate_at_launch = true, - } - ], - ) -} - -resource "aws_lb" "server" { - name = "riju-server" - security_groups = [aws_security_group.alb.id] - subnets = data.aws_subnet_ids.default.ids -} - -resource "aws_lb_target_group" "server_http" { - name = "riju-server-http" - port = 80 - protocol = "HTTP" - vpc_id = data.aws_vpc.default.id -} - -resource "aws_lb_target_group" "server_https" { - name = "riju-server-https" - port = 443 - protocol = "HTTPS" - vpc_id = data.aws_vpc.default.id -} - -output "alb_dns_name" { - value = aws_lb.server.dns_name -} - -output "deploy_aws_access_key_id" { - value = aws_iam_access_key.deploy.id -} - -output "deploy_aws_secret_access_key" { - value = aws_iam_access_key.deploy.secret - sensitive = true -} diff --git a/tf/main.tf b/tf/main.tf new file mode 100644 index 0000000..d32bec5 --- /dev/null +++ b/tf/main.tf @@ -0,0 +1,51 @@ +terraform { + backend "s3" { + key = "state" + region = "us-west-1" + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.45" + } + null = { + source = "hashicorp/null" + version = "~> 3.1" + } + } +} + +data "external" "env" { + program = ["jq", "-n", "env"] +} + +locals { + tags = { + Terraform = "Managed by Terraform" + BillingCategory = "Riju" + } + + ami_available = lookup(data.external.env.result, "AMI_NAME", "") != "" ? true : false +} + +provider "aws" { + region = "us-west-1" + default_tags { + tags = local.tags + } +} + +data "aws_region" "current" {} + +data "aws_vpc" "default" { + default = true +} + +data "aws_subnet_ids" "default" { + vpc_id = data.aws_vpc.default.id +} + +data "aws_subnet" "default" { + for_each = data.aws_subnet_ids.default.ids + id = each.value +} diff --git a/tf/outputs.tf b/tf/outputs.tf new file mode 100644 index 0000000..630adbc --- /dev/null +++ b/tf/outputs.tf @@ -0,0 +1,12 @@ +output "alb_dns_name" { + value = aws_lb.server.dns_name +} + +output "deploy_aws_access_key_id" { + value = aws_iam_access_key.deploy.id +} + +output "deploy_aws_secret_access_key" { + value = aws_iam_access_key.deploy.secret + sensitive = true +} diff --git a/tf/s3.tf b/tf/s3.tf new file mode 100644 index 0000000..67584aa --- /dev/null +++ b/tf/s3.tf @@ -0,0 +1,17 @@ +resource "aws_s3_bucket" "riju" { + bucket = data.external.env.result.S3_BUCKET +} + +resource "aws_s3_bucket_public_access_block" "riju" { + bucket = aws_s3_bucket.riju.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_policy" "riju" { + bucket = aws_s3_bucket.riju.id + policy = data.aws_iam_policy_document.riju.json +} From 7149f817a666aaea2c6030e6a3fba54d88308c0c Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 15:14:26 +0000 Subject: [PATCH 090/104] Many misc updates --- Makefile | 2 +- langs/abc.yaml | 1 + lib/yaml.js | 5 ++-- packer/provision.bash | 11 +++---- supervisor/src/main.go | 4 +-- tf/asg.tf | 8 +++-- tf/iam.tf | 63 +++++++++++++++++++++++++-------------- tf/s3.tf | 34 ++++++++++++++++++++- tools/build-lang-image.js | 11 +++++-- tools/depgraph.js | 8 ++++- tools/packer-build.bash | 2 +- 11 files changed, 108 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 4e4627b..0d4d35d 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ endif # Get rid of 'Entering directory' / 'Leaving directory' messages. MAKE_QUIETLY := MAKELEVEL= make -.PHONY: all $(MAKECMDGOALS) +.PHONY: all $(MAKECMDGOALS) frontend system supervisor all: help diff --git a/langs/abc.yaml b/langs/abc.yaml index 3e97bad..bf8cdb5 100644 --- a/langs/abc.yaml +++ b/langs/abc.yaml @@ -20,6 +20,7 @@ install: repl: | abc input: | + DELAY: 1 WRITE 123 * 234 main: "main.abc" diff --git a/lib/yaml.js b/lib/yaml.js index 1b9a1b7..b68f02b 100644 --- a/lib/yaml.js +++ b/lib/yaml.js @@ -2,6 +2,7 @@ import { promises as fs } from "fs"; import path from "path"; import { validate as validateJSONSchema } from "jsonschema"; +import _ from "lodash"; import YAML from "yaml"; // The build scripts in the language configs assume a specific build @@ -120,7 +121,7 @@ export async function readSharedDepConfig(lang) { // Given a language config JSON, return a list of the Riju shared // dependency names, or an empty list if none are configured for this -// language. +// language. The return value is sorted. export async function getSharedDepsForLangConfig(langConfig) { - return (langConfig.install && langConfig.install.riju) || []; + return [...(langConfig.install && langConfig.install.riju) || []].sort(); } diff --git a/packer/provision.bash b/packer/provision.bash index 046b79e..192dded 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -3,6 +3,7 @@ set -euo pipefail : ${ADMIN_PASSWORD} +: ${AWS_REGION} : ${S3_BUCKET} : ${SUPERVISOR_ACCESS_TOKEN} @@ -23,9 +24,9 @@ ubuntu_name="$(lsb_release -cs)" sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null < fs.readFile(`build/shared/${name}/install.bash`), + )); + const allInstallContents = [].concat.apply([installContents], sharedInstallContents); const hash = await hashDockerfile( "lang", { @@ -41,13 +46,15 @@ async function main() { langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`), sharedHashes: ( await Promise.all( - (await getSharedDepsForLangConfig(await readLangConfig(lang))).map( + sharedDeps.map( async (name) => await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`) ) ) ).sort(), - installHash: crypto.createHash("sha1").update(installContents).digest("hex"), + installHash: allInstallContents.map( + (c) => crypto.createHash("sha1").update(c).digest("hex"), + ).join(""), }, } ); diff --git a/tools/depgraph.js b/tools/depgraph.js index 807ee5b..0760c29 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -124,12 +124,18 @@ async function getImageArtifact({ tag, isBaseImage, isLangImage }) { `build/lang/${isLangImage.lang}/install.bash`, "utf-8" ); + const sharedInstallContents = await Promise.all(isLangImage.sharedDeps.map( + async (name) => fs.readFile(`build/shared/${name}/install.bash`), + )); + const allInstallContents = [].concat.apply([installContents], sharedInstallContents); salt = { langHash: dependencyHashes[`deb:lang-${isLangImage.lang}`], sharedHashes: isLangImage.sharedDeps.map( (name) => dependencyHashes[`deb:shared-${name}`] ), - installHash: crypto.createHash("sha1").update(installContents).digest("hex"), + installHash: allInstallContents.map( + (c) => crypto.createHash("sha1").update(c).digest("hex"), + ).join(""), }; } return await hashDockerfile(name, dependentDockerHashes, { salt }); diff --git a/tools/packer-build.bash b/tools/packer-build.bash index ba77614..f52e12b 100755 --- a/tools/packer-build.bash +++ b/tools/packer-build.bash @@ -4,7 +4,7 @@ set -euo pipefail export AWS_REGION="${AWS_REGION:-$(aws configure get region)}" -if [[ -n "${AWS_REGION}" ]]; then +if [[ -z "${AWS_REGION}" ]]; then echo >&2 "no default AWS region specified, and AWS_REGION unset" exit 1 fi From 80141ce683ca9182538be4c9c16ae15d88ccddbc Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 15:35:54 +0000 Subject: [PATCH 091/104] Mess around with .env handling --- bin/dep | 8 +++++++- bin/docker | 8 +++++++- bin/packer | 13 +++++++++++++ bin/skopeo | 8 +++++++- bin/terraform | 13 +++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) create mode 100755 bin/packer create mode 100755 bin/terraform diff --git a/bin/dep b/bin/dep index 689e1e5..b500f8d 100755 --- a/bin/dep +++ b/bin/dep @@ -2,4 +2,10 @@ set -euo pipefail -exec node "$(dirname "$(realpath "$0")")"/../tools/depgraph.js "$@" +root="$(cd "$(dirname "$0")/.." && echo "${PWD}")" + +set -a +. "${root}/.env" +set +a + +exec node "${root}/tools/depgraph.js" "$@" diff --git a/bin/docker b/bin/docker index d1e1ebb..cc95d60 100755 --- a/bin/docker +++ b/bin/docker @@ -2,7 +2,13 @@ set -euo pipefail -export PATH="$(sed -E 's/:bin:/:/; s/(^|:)bin(:|$)//' <<< "${PATH}")" +root="$(cd "$(dirname "$0")/.." && echo "${PWD}")" + +export PATH="$(sed -E "s_:${root}/bin:_:_; s_(^|:)${root}/bin(:|$)__" <<< "${PATH}")" + +set -a +. "${root}/.env" +set +a if [[ "${OSTYPE:-}" != darwin* ]] && [[ "${EUID}" != 0 ]]; then exec sudo -E docker "$@" diff --git a/bin/packer b/bin/packer new file mode 100755 index 0000000..046bb48 --- /dev/null +++ b/bin/packer @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && echo "${PWD}")" + +export PATH="$(sed -E "s_:${root}/bin:_:_; s_(^|:)${root}/bin(:|$)__" <<< "${PATH}")" + +set -a +. "${root}/.env" +set +a + +exec packer "$@" diff --git a/bin/skopeo b/bin/skopeo index c9a585c..b88d16d 100755 --- a/bin/skopeo +++ b/bin/skopeo @@ -2,7 +2,13 @@ set -euo pipefail -export PATH="$(sed -E 's/:bin:/:/; s/(^|:)bin(:|$)//' <<< "${PATH}")" +root="$(cd "$(dirname "$0")/.." && echo "${PWD}")" + +export PATH="$(sed -E "s_:${root}/bin:_:_; s_(^|:)${root}/bin(:|$)__" <<< "${PATH}")" + +set -a +. "${root}/.env" +set +a if [[ "${OSTYPE:-}" != darwin* ]] && [[ "${EUID}" != 0 ]]; then exec sudo -E skopeo "$@" diff --git a/bin/terraform b/bin/terraform new file mode 100755 index 0000000..58dbe2c --- /dev/null +++ b/bin/terraform @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && echo "${PWD}")" + +export PATH="$(sed -E "s_:${root}/bin:_:_; s_(^|:)${root}/bin(:|$)__" <<< "${PATH}")" + +set -a +. "${root}/.env" +set +a + +exec terraform "$@" From 7da6e0a5f711373dcd83110ddd004fcdd684a29c Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 16:19:47 +0000 Subject: [PATCH 092/104] Infrastructure up and running --- tf/alb.tf | 41 +++++++++++++++++++++++++++++++++++------ tf/asg.tf | 9 ++------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/tf/alb.tf b/tf/alb.tf index b8a271b..0d125bc 100644 --- a/tf/alb.tf +++ b/tf/alb.tf @@ -32,16 +32,45 @@ resource "aws_lb" "server" { subnets = data.aws_subnet_ids.default.ids } -resource "aws_lb_target_group" "server_http" { +resource "aws_lb_target_group" "server" { name = "riju-server-http" port = 80 protocol = "HTTP" vpc_id = data.aws_vpc.default.id } -resource "aws_lb_target_group" "server_https" { - name = "riju-server-https" - port = 443 - protocol = "HTTPS" - vpc_id = data.aws_vpc.default.id +resource "aws_lb_listener" "server_http" { + load_balancer_arn = aws_lb.server.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_lb_listener" "server_https" { + load_balancer_arn = aws_lb.server.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = aws_acm_certificate.riju.arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.server.arn + } +} + +resource "aws_autoscaling_attachment" "server" { + count = local.ami_available ? 1 : 0 + + autoscaling_group_name = aws_autoscaling_group.server[0].name + alb_target_group_arn = aws_lb_target_group.server.arn } diff --git a/tf/asg.tf b/tf/asg.tf index 2b38a6b..e02b089 100644 --- a/tf/asg.tf +++ b/tf/asg.tf @@ -87,19 +87,14 @@ resource "aws_autoscaling_group" "server" { availability_zones = [ for subnet in data.aws_subnet.default : subnet.availability_zone ] - desired_capacity = 0 - min_size = 0 + desired_capacity = 1 + min_size = 1 max_size = 3 launch_template { id = aws_launch_template.server[0].id } - target_group_arns = [ - aws_lb_target_group.server_http.arn, - aws_lb_target_group.server_https.arn, - ] - tags = concat( [ { From 719b16f3c064e8a29e064d61c1d74294de598ddb Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 17:13:59 +0000 Subject: [PATCH 093/104] Infrastructure go brrr --- Makefile | 5 +++-- backend/api.js | 4 ++-- docker/app/Dockerfile | 2 +- langs/unison.yaml | 24 ++++++++++++++++++------ supervisor/src/main.go | 2 +- system/compile.bash | 2 +- tools/depgraph.js | 20 +++++++++++++++++--- tools/generate-deploy-config.js | 21 ++++++++++++++++++++- 8 files changed, 63 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 0d4d35d..6286b1d 100644 --- a/Makefile +++ b/Makefile @@ -224,11 +224,12 @@ upload: # L= T= : Upload .deb to S3 aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null -config: # Generate deployment config file +deploy-config: # Generate deployment config file node tools/generate-deploy-config.js -deploy: # Upload deployment config to S3 +deploy: deploy-config # Upload deployment config to S3 and update ASG instances aws s3 cp $(BUILD)/config.json $(S3_CONFIG) + tools/force-update-asg.bash ### Infrastructure diff --git a/backend/api.js b/backend/api.js index d930d25..689beae 100644 --- a/backend/api.js +++ b/backend/api.js @@ -312,9 +312,9 @@ export class Session { this.send({ event: "terminalClear" }); let cmdline; if (code) { - cmdline = run; + cmdline = `set +e; ${run}`; if (compile) { - cmdline = `( ${compile} ) && ( set +e; ${run} )`; + cmdline = `( ${compile} ) && ( ${run} )`; } } else if (repl) { cmdline = repl; diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index b970056..aa1b828 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /src COPY Makefile ./ COPY system ./system/ -RUN make system +RUN make system UNPRIVILEGED=1 COPY package.json yarn.lock ./ RUN yarn install diff --git a/langs/unison.yaml b/langs/unison.yaml index d9fc00c..e6d1d62 100644 --- a/langs/unison.yaml +++ b/langs/unison.yaml @@ -38,25 +38,37 @@ input: | output: | base.List.reverse +# runProg implementation courtesy of Robert Offner from Unison Slack! main: "main.u" template: | use io - main : '{IO} () - main = 'let + runProg: '{IO, Exception} a -> '{IO} () + runProg f = 'let + printErr err = match err with + Failure _ errMsg _ -> handle putBytes (stdHandle StdErr) (toUtf8 errMsg) with cases + {raise _ -> _} -> () + {_} -> () + match catch f with + Left err -> printErr err + Right _ -> () + + main: '{IO} () + main = runProg 'let printLine "Hello, world!" +createEmpty: "" run: | - echo "Type 'run main' to run the code." + unison -codebase . run.file main.u main + echo "Type 'load main.u' at the repl prompt to bring variables into scope." unison -codebase . -helloInput: | - DELAY: 3 - run main scope: code: | x = 123 * 234 input: | + DELAY: 3 + load main.u DELAY: 3 add x DELAY: 3 diff --git a/supervisor/src/main.go b/supervisor/src/main.go index f284afe..bbd2f49 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -33,8 +33,8 @@ const blueName = "riju-app-blue" const greenName = "riju-app-green" type deploymentConfig struct { - LangImageTags map[string]string `json:"langImageTags"` AppImageTag string `json:"appImageTag"` + LangImageTags map[string]string `json:"langImageTags"` } type supervisorConfig struct { diff --git a/system/compile.bash b/system/compile.bash index 3907274..24bee70 100755 --- a/system/compile.bash +++ b/system/compile.bash @@ -18,7 +18,7 @@ for src in system/src/*.c; do out="${src/src/out}" out="${out/.c}" verbosely clang -Wall -Wextra -Werror -std=c11 "${src}" -o "${out}" - if [[ "${out}" == *-privileged ]]; then + if [[ "${out}" == *-privileged && -z "${UNPRIVILEGED:-}" ]]; then verbosely sudo chown root:riju "${out}" verbosely sudo chmod a=,g=rx,u=rwxs "${out}" fi diff --git a/tools/depgraph.js b/tools/depgraph.js index 0760c29..9c4030c 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -250,9 +250,22 @@ async function getLanguageTestArtifact({ lang }) { }; } -async function getDeployArtifact(langs) { +async function getDeployReadyArtifact(langs) { return { - name: `deploy:prod`, + name: `deploy:ready`, + dependencies: ["image:app"] + .concat(langs.map((lang) => `image:lang-${lang}`)) + .concat(langs.map((lang) => `test:lang-${lang}`)), + publishTarget: true, + publishToRegistry: async () => { + await runCommand(`make deploy-config`); + }, + }; +} + +async function getDeployLiveArtifact(langs) { + return { + name: `deploy:live`, dependencies: ["image:app"] .concat(langs.map((lang) => `image:lang-${lang}`)) .concat(langs.map((lang) => `test:lang-${lang}`)), @@ -298,7 +311,8 @@ async function getDepGraph() { artifacts.push(await getLanguageTestArtifact({ lang: lang })); } artifacts.push(await getImageArtifact({ tag: "app" })); - artifacts.push(await getDeployArtifact(langs)); + artifacts.push(await getDeployReadyArtifact(langs)); + artifacts.push(await getDeployLiveArtifact(langs)); return { informationalDependencies, artifacts }; } diff --git a/tools/generate-deploy-config.js b/tools/generate-deploy-config.js index b3c6d25..a56e437 100644 --- a/tools/generate-deploy-config.js +++ b/tools/generate-deploy-config.js @@ -1,10 +1,29 @@ import { promises as fs } from "fs"; import url from "url"; +import { Command } from "commander"; + +import { getLangs } from "../lib/yaml.js"; +import { getLocalImageLabel } from "./docker-util.js"; + // Get the contents of the JSON file that will be written to S3 in // order to deploy Riju. async function getDeployConfig() { - // TODO + const langs = await getLangs(); + const langImageTags = Object.fromEntries( + await Promise.all( + langs.map(async (lang) => [ + lang, + `lang-${lang}-` + + (await getLocalImageLabel(`riju:lang-${lang}`, "riju.image-hash")), + ]) + ) + ); + const appImageTag = await getLocalImageLabel(`riju:app`, "riju.image-hash"); + return { + appImageTag, + langImageTags, + } } // Parse command-line arguments, run main functionality, and exit. From c354c11f77a618686a0fb44081b3692485276223 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 17:20:23 +0000 Subject: [PATCH 094/104] Automatic reload every 5 minutes --- Makefile | 1 - supervisor/src/main.go | 48 +++++++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 6286b1d..f6495b2 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,6 @@ deploy-config: # Generate deployment config file deploy: deploy-config # Upload deployment config to S3 and update ASG instances aws s3 cp $(BUILD)/config.json $(S3_CONFIG) - tools/force-update-asg.bash ### Infrastructure diff --git a/supervisor/src/main.go b/supervisor/src/main.go index bbd2f49..fa6fe65 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -74,6 +74,27 @@ func (sv *supervisor) status(status string) { log.Println("active: " + status) } +func (sv *supervisor) scheduleReload() string { + uuid := "" + sv.reloadLock.Lock() + if !sv.reloadInProgress { + sv.reloadInProgress = true + sv.reloadUUID = uuidlib.New().String() + uuid = sv.reloadUUID + go sv.reloadWithScheduling() + } else { + if sv.reloadInProgress { + uuid = sv.reloadNextUUID + } else { + sv.reloadNextUUID = uuidlib.New().String() + uuid = sv.reloadNextUUID + } + sv.reloadNeeded = true + } + sv.reloadLock.Unlock() + return uuid +} + func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/api/supervisor") { if r.URL.Path == "/api/supervisor/v1/reload" { @@ -81,23 +102,7 @@ func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) return } - uuid := "" - sv.reloadLock.Lock() - if !sv.reloadInProgress { - sv.reloadInProgress = true - sv.reloadUUID = uuidlib.New().String() - uuid = sv.reloadUUID - go sv.reloadWithScheduling() - } else { - if sv.reloadInProgress { - uuid = sv.reloadNextUUID - } else { - sv.reloadNextUUID = uuidlib.New().String() - uuid = sv.reloadNextUUID - } - sv.reloadNeeded = true - } - sv.reloadLock.Unlock() + uuid := sv.scheduleReload() fmt.Fprintln(w, uuid) return } @@ -179,6 +184,14 @@ func (sv *supervisor) reloadWithScheduling() { sv.reloadUUID = sv.reloadNextUUID sv.reloadNextUUID = "" go sv.reloadWithScheduling() + } else { + go func() { + // Arguably slightly incorrect but it's fine + // if we reload slightly more than once per 5 + // minutes. + time.Sleep(5 * time.Minute) + sv.scheduleReload() + }() } sv.reloadLock.Unlock() } @@ -389,6 +402,7 @@ func main() { awsAccountNumber: *ident.Account, reloadJobs: map[string]*reloadJob{}, } + go sv.scheduleReload() log.Println("listening on http://0.0.0.0:80") log.Fatalln(http.ListenAndServe("0.0.0.0:80", sv)) } From fc0fcff7a6b5499665279a745a48b3753f149d44 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 18:22:50 +0000 Subject: [PATCH 095/104] Ensure .deb compression --- Makefile | 1 + tools/ensure-deb-compressed.bash | 76 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100755 tools/ensure-deb-compressed.bash diff --git a/Makefile b/Makefile index f6495b2..e5060f0 100644 --- a/Makefile +++ b/Makefile @@ -220,6 +220,7 @@ push: # I= : Push Riju image to Docker registry upload: # L= T= : Upload .deb to S3 @: $${L} $${T} $${S3_BUCKET} + tools/ensure-deb-compressed.bash aws s3 rm --recursive $(S3_HASH) aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null diff --git a/tools/ensure-deb-compressed.bash b/tools/ensure-deb-compressed.bash new file mode 100755 index 0000000..757a942 --- /dev/null +++ b/tools/ensure-deb-compressed.bash @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: ${L} ${T} + +deb="build/${T}/${L}/riju-${T}-${L}.deb" +tmp="build/${T}/${L}/ar-tmp" + +rm -rf "${tmp}" + +files="$(ar t "${deb}" | grep -F .tar | grep -Fv .xz || true)" + +if [[ -z "${files}" ]]; then + exit 0 +fi + +du -sh "${deb}" + +echo >&2 "found files in ${deb} needing recompression:" +echo "${files}" | sed >&2 's/^/ /' + +echo >&2 "extracting ${deb} to ${tmp}..." +mkdir -p "${tmp}" +ar xv "${deb}" --output "${tmp}" | sed >&2 's/^/ /' + +echo >&2 "recompressing extracted files..." + +processed_files=() +while read file; do + if echo "${file}" | grep -F .tar | grep -Fvq .xz; then + ext="$(echo "${file}" | sed -E 's/^.+\.tar//')" + case "${ext}" in + "") + echo >&2 " ${file} => ${file}.xz" + xz "${tmp}/${file}" + processed_files=("${processed_files[@]}" "${file}.xz") + ;; + ".gz") + echo >&2 " ${file} => ${file%.gz}.xz" + gunzip "${tmp}/${file}" -c | xz - > "${tmp}/${file%.gz}" + rm "${tmp}/${file}" + processed_files=("${processed_files[@]}" "${file%.gz}") + ;; + *) + echo >&2 " !! unknown file extension in ${file}" + exit 1 + ;; + esac + else + processed_files=("${processed_files[@]}" "${file}") + fi +done < <(ar t "${deb}") + +# NB we need to keep track of the processed_files array because the +# order needs to be preserved, surprisingly enough: +# https://ubuntuforums.org/archive/index.php/t-1481153.html + +echo >&2 "recreating archive at ${deb}.tmp..." +pushd "${tmp}" >/dev/null +ar rcv "../$(basename "${deb}").tmp" "${processed_files[@]}" | sed 's/^/ /' +popd >/dev/null + +echo >&2 "cleaning up ${tmp}..." +rm -rf "${tmp}" + +echo >&2 "renaming ${deb}.tmp => ${deb}..." +mv "${deb}.tmp" "${deb}" + +files="$(ar t "${deb}" | grep -F .tar | grep -Fv .xz || true)" + +if [[ -n "${files}" ]]; then + echo >&2 "error: still found files in ${deb} needing recompression:" + echo "${files}" | sed >&2 's/^/ /' + exit 1 +fi From 478e230d69a24fe43c880c2992c03717b3bd7712 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 20:09:31 +0000 Subject: [PATCH 096/104] Require auth token for supervisor API --- supervisor/src/main.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/supervisor/src/main.go b/supervisor/src/main.go index fa6fe65..4d34313 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -97,6 +97,19 @@ func (sv *supervisor) scheduleReload() string { func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/api/supervisor") { + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + http.Error(w, "401 Authorization header missing", http.StatusUnauthorized) + return + } + if !strings.HasPrefix(authHeader, "Bearer ") { + http.Error(w, "401 malformed Authorization header", http.StatusUnauthorized) + return + } + if authHeader != "Bearer " + sv.config.AccessToken { + http.Error(w, "401 wrong access token", http.StatusUnauthorized) + return + } if r.URL.Path == "/api/supervisor/v1/reload" { if r.Method != http.MethodPost { http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) @@ -111,8 +124,8 @@ func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) return } - uuids := r.URL.Query()["uuid"] - if len(uuids) == 0 { + uuid := r.URL.Query().Get("uuid") + if uuid == "" { http.Error( w, "400 missing uuid query parameter", @@ -120,15 +133,6 @@ func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { ) return } - if len(uuids) > 1 { - http.Error( - w, - "400 more than one uuid query parameter", - http.StatusBadRequest, - ) - return - } - uuid := uuids[0] sv.reloadLock.Lock() job := sv.reloadJobs[uuid] if job == nil { From 8f619b69c6456c3afad11e8463a987faaf2f0d5a Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 4 Jul 2021 23:37:08 +0000 Subject: [PATCH 097/104] Getting things spun up --- packer/provision.bash | 7 ++++++- supervisor/src/main.go | 22 ++++++++++++++++++++++ tf/asg.tf | 6 +++++- tools/generate-deploy-config.js | 2 +- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/packer/provision.bash b/packer/provision.bash index 192dded..187dd4b 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -7,6 +7,11 @@ set -euo pipefail : ${S3_BUCKET} : ${SUPERVISOR_ACCESS_TOKEN} +# I think there is a race condition related to Ubuntu wanting to do an +# automated system upgrade at boot, which causes 'apt-get update' to +# sometimes fail with an obscure error message. +sleep 5 + mkdir /tmp/riju-work pushd /tmp/riju-work @@ -50,7 +55,7 @@ sudo hostnamectl set-hostname riju sudo systemctl enable riju -sudo passwd -l ubuntu +sudo userdel ubuntu -f popd rm -rf /tmp/riju-work diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 4d34313..42bd8a6 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "regexp" + "sort" "strings" "sync" "time" @@ -221,6 +222,7 @@ func (sv *supervisor) reload() error { "docker", "image", "ls", "--format", "{{ .Repository }}:{{ .Tag }}", ) + dockerImageLs.Stderr = os.Stderr out, err := dockerImageLs.Output() if err != nil { return err @@ -237,6 +239,7 @@ func (sv *supervisor) reload() error { neededTags = append(neededTags, tag) } neededTags = append(neededTags, deployCfg.AppImageTag) + sort.Strings(neededTags) for _, tag := range neededTags { if !existingTags[tag] { sv.status("pulling image for " + tag) @@ -247,6 +250,8 @@ func (sv *supervisor) reload() error { tag, ) dockerPull := exec.Command("docker", "pull", fullImage) + dockerPull.Stdout = os.Stdout + dockerPull.Stderr = os.Stderr if err := dockerPull.Run(); err != nil { return err } @@ -254,6 +259,8 @@ func (sv *supervisor) reload() error { "docker", "tag", fullImage, fmt.Sprintf("riju:%s", tag), ) + dockerTag.Stdout = os.Stdout + dockerTag.Stderr = os.Stderr if err := dockerTag.Run(); err != nil { return err } @@ -265,12 +272,15 @@ func (sv *supervisor) reload() error { } var port int var name string + var oldName string if sv.isGreen { port = bluePort name = blueName + oldName = greenName } else { port = greenPort name = greenName + oldName = blueName } sv.status("starting container " + name) dockerRun := exec.Command( @@ -283,6 +293,8 @@ func (sv *supervisor) reload() error { "--name", name, fmt.Sprintf("riju:%s", deployCfg.AppImageTag), ) + dockerRun.Stdout = os.Stdout + dockerRun.Stderr = os.Stderr dockerRun.Env = append(os.Environ(), fmt.Sprintf("RIJU_DEPLOY_CONFIG=%s", deployCfgStr)) if err := dockerRun.Run(); err != nil { return err @@ -303,6 +315,13 @@ func (sv *supervisor) reload() error { return errors.New("container did not appear to be healthy") } sv.isGreen = !sv.isGreen + sv.status("stopping old container") + dockerStop := exec.Command("docker", "stop", oldName) + dockerStop.Stdout = dockerStop.Stdout + dockerStop.Stderr = dockerStop.Stderr + if err := dockerStop.Run(); err != nil { + return err + } sv.status("reload complete") return nil } @@ -339,6 +358,7 @@ func main() { "docker", "container", "ls", "--format", "{{ .Names }}:{{ .CreatedAt }}", ) + dockerContainerLs.Stderr = os.Stderr out, err := dockerContainerLs.Output() if err != nil { log.Fatalln(err) @@ -391,6 +411,8 @@ func main() { } log.Printf("stopping %s container as it is newer\n", color) dockerStop := exec.Command("docker", "stop", name) + dockerStop.Stdout = os.Stdout + dockerStop.Stderr = os.Stderr if err := dockerStop.Run(); err != nil { log.Fatalln(err) } diff --git a/tf/asg.tf b/tf/asg.tf index e02b089..925ee94 100644 --- a/tf/asg.tf +++ b/tf/asg.tf @@ -110,5 +110,9 @@ resource "aws_autoscaling_group" "server" { propagate_at_launch = true, } ], - ) + ) + + lifecycle { + ignore_changes = [target_group_arns] + } } diff --git a/tools/generate-deploy-config.js b/tools/generate-deploy-config.js index a56e437..bb66f51 100644 --- a/tools/generate-deploy-config.js +++ b/tools/generate-deploy-config.js @@ -19,7 +19,7 @@ async function getDeployConfig() { ]) ) ); - const appImageTag = await getLocalImageLabel(`riju:app`, "riju.image-hash"); + const appImageTag = `app-` + await getLocalImageLabel(`riju:app`, "riju.image-hash"); return { appImageTag, langImageTags, From 32210f52b024c113d4a5d08444b0aa24e5d85b33 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 00:05:32 +0000 Subject: [PATCH 098/104] Awesome --- supervisor/go.mod | 1 + supervisor/go.sum | 2 ++ supervisor/src/main.go | 45 ++++++++++++++++++++++++++++++++++++++++++ tf/iam.tf | 21 ++++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/supervisor/go.mod b/supervisor/go.mod index 116938f..3021b2a 100644 --- a/supervisor/go.mod +++ b/supervisor/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/config v1.4.1 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.4.0 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.11.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.5.0 // indirect github.com/caarlos0/env/v6 v6.6.2 // indirect diff --git a/supervisor/go.sum b/supervisor/go.sum index ed79512..5bd5fe1 100644 --- a/supervisor/go.sum +++ b/supervisor/go.sum @@ -10,6 +10,8 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.1 h1:ag1MjvYmE8hnvl2/3LYOog github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.1/go.mod h1:WXrj1wxGcYFfQ6H4xqsbVziISWQT55SlpX8B5+EqLOw= github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0 h1:DJq/vXXF+LAFaa/kQX9C6arlf4xX4uaaqGWIyAKOCpM= github.com/aws/aws-sdk-go-v2/internal/ini v1.1.0/go.mod h1:qGQ/9IfkZonRNSNLE99/yBJ7EPA/h8jlWEqtJCcaj+Q= +github.com/aws/aws-sdk-go-v2/service/ecr v1.4.0 h1:cgMcR4Y2JFhWHFDNiVYLApc5kSaGK0geqqL/2XvP77M= +github.com/aws/aws-sdk-go-v2/service/ecr v1.4.0/go.mod h1:66eKvbrtxgZWfVHNwdncN8vciDvc00gX2flcATKqLYQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.0 h1:wfI4yrOCMAGdHaEreQ65ycSmPLVc2Q82O+r7ZxYTynA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.0/go.mod h1:2Kc2Pybp1Hr2ZCCOz78mWnNSZYEKKBQgNcizVGk9sko= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.0 h1:g2npzssI/6XsoQaPYCxliMFeC5iNKKvO0aC+/wWOE0A= diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 42bd8a6..33bf915 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -21,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" awsConfig "github.com/aws/aws-sdk-go-v2/config" s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/caarlos0/env/v6" @@ -59,6 +62,7 @@ type supervisor struct { awsAccountNumber string awsRegion string s3 *s3.Client + ecr *ecr.Client reloadLock sync.Mutex reloadInProgress bool @@ -204,6 +208,46 @@ func (sv *supervisor) reloadWithScheduling() { var rijuImageRegexp = regexp.MustCompile(`(?:^|/)riju:([^<>]+)$`) func (sv *supervisor) reload() error { + sv.status("getting access token from ECR") + ecrResp, err := sv.ecr.GetAuthorizationToken( + context.Background(), + &ecr.GetAuthorizationTokenInput{}, + ) + if err != nil { + return err + } + if len(ecrResp.AuthorizationData) != 1 { + return fmt.Errorf( + "got unexpected number (%d) of authorization tokens", + len(ecrResp.AuthorizationData), + ) + } + authInfo, err := base64.StdEncoding.DecodeString(*ecrResp.AuthorizationData[0].AuthorizationToken) + if err != nil { + return err + } + authInfoParts := strings.Split(string(authInfo), ":") + if len(authInfoParts) != 2 { + return errors.New("got malformed auth info from ECR") + } + dockerUsername := authInfoParts[0] + dockerPassword := authInfoParts[1] + sv.status("authenticating Docker client to ECR") + dockerLogin := exec.Command( + "docker", "login", + "--username", dockerUsername, + "--password-stdin", + fmt.Sprintf( + "%s.dkr.ecr.%s.amazonaws.com", + sv.awsAccountNumber, sv.awsRegion, + ), + ) + dockerLogin.Stdin = bytes.NewReader([]byte(dockerPassword)) + dockerLogin.Stdout = os.Stdout + dockerLogin.Stderr = os.Stderr + if err := dockerLogin.Run(); err != nil { + return err + } sv.status("downloading deployment config from S3") dl := s3manager.NewDownloader(sv.s3) buf := s3manager.NewWriteAtBuffer([]byte{}) @@ -424,6 +468,7 @@ func main() { greenProxyHandler: httputil.NewSingleHostReverseProxy(greenUrl), isGreen: isGreen, s3: s3.NewFromConfig(awsCfg), + ecr: ecr.NewFromConfig(awsCfg), awsRegion: awsCfg.Region, awsAccountNumber: *ident.Account, reloadJobs: map[string]*reloadJob{}, diff --git a/tf/iam.tf b/tf/iam.tf index 88a5ec7..21d4a2a 100644 --- a/tf/iam.tf +++ b/tf/iam.tf @@ -49,6 +49,27 @@ data "aws_iam_policy_document" "server" { "arn:aws:s3:::${aws_s3_bucket.riju.bucket}/config.json", ] } + + statement { + actions = [ + "ecr:GetAuthorizationToken", + ] + + resources = [ + "*", + ] + } + + statement { + actions = [ + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ] + + resources = [ + aws_ecr_repository.riju.arn, + ] + } } resource "aws_iam_policy" "server" { From 3b82fd333f2223dcadb24a88724111a5c38a2898 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 00:11:33 +0000 Subject: [PATCH 099/104] Init volume at supervisor startup --- supervisor/src/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 33bf915..c7e7674 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -378,6 +378,13 @@ func main() { log.Fatalln(err) } + rijuInitVolume := exec.Command("riju-init-volume") + rijuInitVolume.Stdout = rijuInitVolume.Stdout + rijuInitVolume.Stderr = rijuInitVolume.Stderr + if err := rijuInitVolume.Run(); err != nil { + log.Fatalln(err) + } + blueUrl, err := url.Parse(fmt.Sprintf("http://localhost:%d", bluePort)) if err != nil { log.Fatalln(err) From ab61f0eb4e9ef6bc7ca1d857b580b6d1cf86aba8 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 00:41:53 +0000 Subject: [PATCH 100/104] Setting up CI --- .circleci/config.yml | 20 -------------------- .github/workflows/main.yml | 21 +++++++++++++++++++++ bin/packer | 2 ++ bin/terraform | 2 ++ tf/asg.tf | 2 +- tools/ci-bootstrap.bash | 4 ---- 6 files changed, 26 insertions(+), 25 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/main.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 831cfae..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2 -workflows: - version: 2 - ci: - jobs: - - build_and_deploy: - filters: - branches: - only: master - tags: - ignore: /.*/ -jobs: - build_and_deploy: - machine: - image: ubuntu-2004:202010-01 - steps: - - checkout - - run: - command: tools/ci-bootstrap.bash - no_output_timeout: 2h diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..ce0dcac --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,21 @@ +name: Build and deploy +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build and deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_REPO: 084011155226.dkr.ecr.us-west-1.amazonaws.com/riju + S3_BUCKET: riju + run: | + make image shell I=ci CMD="tools/ci-bootstrap.bash" diff --git a/bin/packer b/bin/packer index 046bb48..8371d78 100755 --- a/bin/packer +++ b/bin/packer @@ -10,4 +10,6 @@ set -a . "${root}/.env" set +a +cd "${root}/packer" + exec packer "$@" diff --git a/bin/terraform b/bin/terraform index 58dbe2c..b4cf3b2 100755 --- a/bin/terraform +++ b/bin/terraform @@ -10,4 +10,6 @@ set -a . "${root}/.env" set +a +cd "${root}/tf" + exec terraform "$@" diff --git a/tf/asg.tf b/tf/asg.tf index 925ee94..ca48c4a 100644 --- a/tf/asg.tf +++ b/tf/asg.tf @@ -63,7 +63,7 @@ resource "aws_launch_template" "server" { device_name = "/dev/sdh" ebs { volume_type = "gp3" - volume_size = 125 + volume_size = 256 } } diff --git a/tools/ci-bootstrap.bash b/tools/ci-bootstrap.bash index 5128d41..1cfec51 100755 --- a/tools/ci-bootstrap.bash +++ b/tools/ci-bootstrap.bash @@ -4,11 +4,7 @@ set -euo pipefail : ${AWS_ACCESS_KEY_ID} : ${AWS_SECRET_ACCESS_KEY} -: ${DEPLOY_SSH_PRIVATE_KEY} -: ${DOCKER_PASSWORD} : ${DOCKER_REPO} -: ${DOCKER_USERNAME} -: ${DOMAIN} : ${S3_BUCKET} make image shell I=ci CMD="tools/ci-run.bash" From 9c62bb32130ca5117ed9d9b32bf9fd66d0f495c1 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 00:52:03 +0000 Subject: [PATCH 101/104] Container should be able to start --- {tools => lib}/jsonschema.yaml | 0 lib/yaml.js | 2 +- supervisor/src/main.go | 22 +++++++++++----------- 3 files changed, 12 insertions(+), 12 deletions(-) rename {tools => lib}/jsonschema.yaml (100%) diff --git a/tools/jsonschema.yaml b/lib/jsonschema.yaml similarity index 100% rename from tools/jsonschema.yaml rename to lib/jsonschema.yaml diff --git a/lib/yaml.js b/lib/yaml.js index b68f02b..e89a90d 100644 --- a/lib/yaml.js +++ b/lib/yaml.js @@ -15,7 +15,7 @@ import YAML from "yaml"; // * we are using bash with 'set -euxo pipefail' async function readJSONSchemaFromDisk() { - return YAML.parse(await fs.readFile("tools/jsonschema.yaml", "utf-8"), { + return YAML.parse(await fs.readFile("lib/jsonschema.yaml", "utf-8"), { merge: true, }); } diff --git a/supervisor/src/main.go b/supervisor/src/main.go index c7e7674..f38d090 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -331,7 +331,7 @@ func (sv *supervisor) reload() error { "docker", "run", "-d", "-v", "/var/run/riju:/var/run/riju", "-v", "/var/run/docker.sock:/var/run/docker.sock", - "-p", fmt.Sprintf("%s:6119", port), + "-p", fmt.Sprintf("127.0.0.1:%d:6119", port), "-e", "RIJU_DEPLOY_CONFIG", "-e", "ANALYTICS=1", "--name", name, @@ -344,7 +344,7 @@ func (sv *supervisor) reload() error { return err } sv.status("waiting for container to start up") - time.Sleep(5) + time.Sleep(5 * time.Second) sv.status("checking that container is healthy") resp, err := http.Get(fmt.Sprintf("http://localhost:%d", port)) if err != nil { @@ -360,10 +360,10 @@ func (sv *supervisor) reload() error { } sv.isGreen = !sv.isGreen sv.status("stopping old container") - dockerStop := exec.Command("docker", "stop", oldName) - dockerStop.Stdout = dockerStop.Stdout - dockerStop.Stderr = dockerStop.Stderr - if err := dockerStop.Run(); err != nil { + dockerRm := exec.Command("docker", "rm", "-f", oldName) + dockerRm.Stdout = dockerRm.Stdout + dockerRm.Stderr = dockerRm.Stderr + if err := dockerRm.Run(); err != nil { return err } sv.status("reload complete") @@ -406,7 +406,7 @@ func main() { } dockerContainerLs := exec.Command( - "docker", "container", "ls", + "docker", "container", "ls", "-a", "--format", "{{ .Names }}:{{ .CreatedAt }}", ) dockerContainerLs.Stderr = os.Stderr @@ -461,10 +461,10 @@ func main() { name = greenName } log.Printf("stopping %s container as it is newer\n", color) - dockerStop := exec.Command("docker", "stop", name) - dockerStop.Stdout = os.Stdout - dockerStop.Stderr = os.Stderr - if err := dockerStop.Run(); err != nil { + dockerRm := exec.Command("docker", "rm", "-f", name) + dockerRm.Stdout = os.Stdout + dockerRm.Stderr = os.Stderr + if err := dockerRm.Run(); err != nil { log.Fatalln(err) } } From bd526e046198b8a4221516ba5a241e3b719e4268 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 01:25:55 +0000 Subject: [PATCH 102/104] More shenanigans --- Makefile | 13 ++++++++++--- tf/ecr.tf | 5 +++++ tf/main.tf | 10 ++++++++++ tools/ci-run.bash | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e5060f0..61934f7 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,10 @@ S3_HASH := $(S3)/hashes/riju-$(T)-$(L) S3_CONFIG := $(S3)/config.json ifneq ($(CMD),) +C_CMD := -c '$(CMD)' BASH_CMD := bash -c '$(CMD)' else +C_CMD := BASH_CMD := endif @@ -96,7 +98,8 @@ else endif ecr: # Authenticate to ECR (temporary credentials) - id="$$(aws sts get-caller-identity | jq .Account -r)"; aws ecr get-login-password --region us-west-1 | docker login --username AWS --password-stdin "$${id}.dkr.ecr.us-west-1.amazonaws.com" + aws ecr get-login-password | docker login --username AWS --password-stdin $(subst /riju,,$(DOCKER_REPO)) + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(subst /riju,,$(PUBLIC_DOCKER_REPO)) ### Build packaging scripts @@ -215,6 +218,10 @@ push: # I= : Push Riju image to Docker registry @: $${I} $${DOCKER_REPO} docker tag riju:$(I) $(DOCKER_REPO):$(I)-$(IMAGE_HASH) docker push $(DOCKER_REPO):$(I)-$(IMAGE_HASH) +ifeq ($(I),ubuntu) + docker tag riju:$(I) $(PUBLIC_DOCKER_REPO):$(I) + docker push $(PUBLIC_DOCKER_REPO):$(I) +endif docker tag riju:$(I) $(DOCKER_REPO):$(I) docker push $(DOCKER_REPO):$(I) @@ -248,8 +255,8 @@ dockerignore: # Update .dockerignore from .gitignore and .dockerignore.in ## manual commands (Docker, Terraform, Packer, etc.) directly, as ## opposed to through the Makefile. -env: # Run shell with .env file loaded and $PATH fixed - exec bash +env: # [CMD=] : Run shell with .env file loaded and $PATH fixed + exec bash $(C_CMD) tmux: # Start or attach to tmux session MAKELEVEL= tmux attach || MAKELEVEL= tmux new-session -s tmux diff --git a/tf/ecr.tf b/tf/ecr.tf index 9202998..fd799d4 100644 --- a/tf/ecr.tf +++ b/tf/ecr.tf @@ -2,3 +2,8 @@ resource "aws_ecr_repository" "riju" { name = "riju" image_tag_mutability = "IMMUTABLE" } + +resource "aws_ecrpublic_repository" "riju" { + provider = aws.us_east_1 + repository_name = "riju" +} diff --git a/tf/main.tf b/tf/main.tf index d32bec5..8be2123 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -35,6 +35,16 @@ provider "aws" { } } +provider "aws" { + alias = "us_east_1" + region = "us-east-1" + default_tags { + tags = local.tags + } +} + +data "aws_caller_identity" "current" {} + data "aws_region" "current" {} data "aws_vpc" "default" { diff --git a/tools/ci-run.bash b/tools/ci-run.bash index 195412f..df8a00a 100755 --- a/tools/ci-run.bash +++ b/tools/ci-run.bash @@ -5,4 +5,4 @@ set -euo pipefail echo "${DOCKER_PASSWORD}" | sudo -E docker login --username "${DOCKER_USERNAME}" --password-stdin make system -make publish Z=xz CI=1 TEST_PATIENCE=4 TEST_CONCURRENCY=1 +make env CMD="dep deploy:live --publish --all" Z=xz CI=1 TEST_PATIENCE=4 TEST_CONCURRENCY=1 From bd831fa9312947f8a82113410dfb3899c8610553 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 05:37:03 +0000 Subject: [PATCH 103/104] Fix silly error, make ECR repo mutable --- supervisor/src/main.go | 2 +- tf/ecr.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/supervisor/src/main.go b/supervisor/src/main.go index f38d090..48e1b5d 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -421,7 +421,7 @@ func main() { if match := rijuContainerRegexp.FindStringSubmatch(line); match != nil { name := match[1] created, err := time.Parse( - "2006-01-02 15:04:05 -0070 MST", + "2006-01-02 15:04:05 -0700 MST", match[2], ) if err != nil { diff --git a/tf/ecr.tf b/tf/ecr.tf index fd799d4..0ff28e6 100644 --- a/tf/ecr.tf +++ b/tf/ecr.tf @@ -1,6 +1,6 @@ resource "aws_ecr_repository" "riju" { name = "riju" - image_tag_mutability = "IMMUTABLE" + image_tag_mutability = "MUTABLE" } resource "aws_ecrpublic_repository" "riju" { From c85ed6d586a763d2c2dba692efbd147fa8bf6ba4 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 5 Jul 2021 14:42:37 +0000 Subject: [PATCH 104/104] Support RIJU_DEPLOY_CONFIG --- backend/util.js | 29 ++++++++++++++++++++++++++++- system/src/riju-system-privileged.c | 23 ++++++++++++++++++----- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/backend/util.js b/backend/util.js index 52a2b2a..dcf156c 100644 --- a/backend/util.js +++ b/backend/util.js @@ -4,6 +4,29 @@ import process from "process"; import { v4 as getUUIDOrig } from "uuid"; +function computeImageHashes() { + let deployConfig = process.env.RIJU_DEPLOY_CONFIG; + if (!deployConfig) + return {}; + deployConfig = JSON.parse(deployConfig); + const imageHashes = {}; + for (const [lang, tag] of Object.entries(deployConfig.langImageTags)) { + const prefix = `lang-${lang}-` + if (!tag.startsWith(prefix)) { + throw new Error(`malformed tag ${tag}`); + } + const imageHash = tag.slice(prefix.length); + if (imageHash.length !== 40) { + throw new Error(`malformed tag ${tag}`); + } + imageHashes[lang] = imageHash; + } + console.log(imageHashes); + return imageHashes; +} + +const imageHashes = computeImageHashes(); + export function quote(str) { return "'" + str.replace(/'/g, `'"'"'`) + "'"; } @@ -48,7 +71,11 @@ export async function run(args, log, options) { } export function privilegedSession({ uuid, lang }) { - return [rijuSystemPrivileged, "session", uuid, lang]; + const cmdline = [rijuSystemPrivileged, "session", uuid, lang]; + if (imageHashes[lang]) { + cmdline.push(imageHashes[lang]); + } + return cmdline; } export function privilegedExec({ uuid }, args) { diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 493c9d3..4b976fc 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -22,7 +22,7 @@ void __attribute__ ((noreturn)) die(char *msg) void die_with_usage() { die("usage:\n" - " riju-system-privileged session UUID LANG\n" + " riju-system-privileged session UUID LANG [IMAGE-HASH]\n" " riju-system-privileged exec UUID CMDLINE...\n" " riju-system-privileged pty UUID CMDLINE..."); } @@ -44,16 +44,28 @@ char *parseLang(char *lang) { return lang; } +char *parseImageHash(char *imageHash) +{ + if (strnlen(imageHash, 41) != 40) + die("illegal imageHash"); + for (char *ptr = imageHash; *ptr; ++ptr) + if (!((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= '0' && *ptr <= '9'))) + die("illegal imageHash"); + return imageHash; +} + void wait_alarm(int signum) { (void)signum; die("container did not come up within 1 second"); } -void session(char *uuid, char *lang) +void session(char *uuid, char *lang, char *imageHash) { char *image, *container, *hostname, *volume, *fifo; - if (asprintf(&image, "riju:lang-%s", lang) < 0) + if ((imageHash != NULL ? + asprintf(&image, "riju:lang-%s-%s", lang, imageHash) : + asprintf(&image, "riju:lang-%s", lang)) < 0) die("asprintf failed"); if (asprintf(&container, "riju-session-%s", uuid) < 0) die("asprintf failed"); @@ -157,11 +169,12 @@ int main(int argc, char **argv) if (argc < 2) die_with_usage(); if (!strcmp(argv[1], "session")) { - if (argc != 4) + if (argc < 4 || argc > 5) die_with_usage(); char *uuid = parseUUID(argv[2]); char *lang = parseLang(argv[3]); - session(uuid, lang); + char *imageHash = argc == 5 ? parseImageHash(argv[4]) : NULL; + session(uuid, lang, imageHash); return 0; } if (!strcmp(argv[1], "exec")) {