Miscellaneous work starting to support langs

This commit is contained in:
Radon Rosborough 2021-01-06 21:44:27 -08:00
parent 6450df047f
commit 9a99429f48
29 changed files with 222 additions and 120 deletions

View File

@ -12,64 +12,75 @@ S3_DEBS := s3://$(S3_BUCKET)-debs
S3_DEB := $(S3_DEBS)/debs/$(DEB)
S3_HASH := $(S3_DEBS)/hashes/riju-$(T)-$(L)
.PHONY: help
.PHONY: all $(MAKECMDGOALS)
help:
@echo "usage:"
@echo
@cat Makefile | \
grep -E '[.]PHONY|[#]##' | \
sed -E 's/[.]PHONY: */ make /' | \
grep -E '^[^.:[:space:]]+:|[#]##' | \
sed -E 's/([^.:[:space:]]+):.*/ make \1/' | \
sed -E 's/[#]## *(.+)/\n (\1)\n/'
### Build artifacts locally
.PHONY: image
ifneq ($(NC),)
NO_CACHE := --no-cache
else
NO_CACHE :=
endif
image:
@: $${I}
ifeq ($(I),composite)
node tools/build-composite-image.js
else ifneq (,$(filter $(I),admin ci))
docker build . -f docker/$(I)/Dockerfile -t riju:$(I)
docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
else
hash="$$(node tools/hash-dockerfile.js $(I) | grep .)"; docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --label riju.image-hash="$${hash}"
hash="$$(node tools/hash-dockerfile.js $(I) | grep .)"; docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --label riju.image-hash="$${hash}" $(NO_CACHE)
endif
.PHONY: script
script:
@: $${L} $${T}
mkdir -p $(BUILD)
node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash
chmod +x $(BUILD)/build.bash
.PHONY: scripts
scripts:
@: $${L}
node tools/make-foreach.js --types script L=$(L)
.PHONY: all-scripts
all-scripts:
node tools/make-foreach.js --pkgs script
.PHONY: pkg
pkg:
pkg-clean:
@: $${L} $${T}
rm -rf $(BUILD)/src $(BUILD)/pkg
mkdir -p $(BUILD)/src $(BUILD)/pkg
cd $(BUILD)/src && pkg="$(PWD)/$(BUILD)/pkg" ../build.bash
pkg-build:
@: $${L} $${T}
cd $(BUILD)/src && pkg="$(PWD)/$(BUILD)/pkg" $(or $(CMD),../build.bash)
pkg-debug:
@: $${L} $${T}
make pkg-build L=$(L) T=$(T) CMD=bash
pkg-deb:
@: $${L} $${T}
fakeroot dpkg-deb --build $(BUILD)/pkg $(BUILD)/$(DEB)
.PHONY: pkgs
pkg: pkg-clean pkg-build pkg-deb
pkgs:
@: $${L}
node tools/make-foreach.js --types pkg L=$(L)
.PHONY: repkg
repkg:
repkg: script
@: $${L} $${T}
make script L=$(L) T=$(T)
make shell I=packaging CMD="make pkg L=$(L) T=$(T)"
ctr="$$(docker container ls -f label="riju-install-target=yes" -l -q)"; test "$${ctr}" || (echo "no valid container is live"; exit 1); docker exec "$${ctr}" make install L=$(L) T=$(T)
.PHONY: repkgs
repkgs:
@: $${L}
node tools/make-foreach.js --types repkg L=$(L)
@ -87,7 +98,6 @@ else
SHELL_PORTS :=
endif
.PHONY: shell
shell:
@: $${I}
ifneq (,$(filter $(I),admin ci))
@ -100,106 +110,91 @@ else
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) riju:$(I) $(CMD)
endif
.PHONY: install
install:
@: $${L} $${T}
if [[ -z "$$(ls -A /var/lib/apt/lists)" ]]; then sudo apt update; fi
sudo apt reinstall -y ./$(BUILD)/$(DEB)
.PHONY: installs
installs:
@: $${L}
node tools/make-foreach.js --types install L=$(L)
### Build and run application code
.PHONY: frontend
frontend:
npx webpack --mode=production
.PHONY: frontend-dev
frontend-dev:
watchexec -w webpack.config.cjs -w node_modules -r --no-environment -- "echo 'Running webpack...' >&2; npx webpack --mode=development --watch"
.PHONY: system
system:
./system/compile.bash
.PHONY: system-dev
system-dev:
watchexec -w system/src -n -- ./system/compile.bash
.PHONY: server
server:
node backend/server.js
.PHONY: server-dev
server-dev:
watchexec -w backend -r -n -- node backend/server.js
.PHONY: build
build: frontend system
.PHONY: dev
dev:
make -j3 frontend-dev system-dev server-dev
.PHONY: test
test:
node backend/test-runner.js $(F)
sandbox:
node backend/sandbox.js
### Fetch artifacts from registries
.PHONY: pull-base
pull-base:
docker pull ubuntu:rolling
.PHONY: pull
pull:
@: $${I} $${DOCKER_REPO}
docker pull $(DOCKER_REPO):$(I)
docker tag $(DOCKER_REPO):$(I) riju:$(I)
.PHONY: download
download:
@: $${L} $${T} $${S3_BUCKET}
mkdir -p $(BUILD)
aws s3 cp $(S3_DEB) $(BUILD)/$(DEB)
.PHONY: plan
plan:
node tools/plan-publish.js
.PHONY: sync
sync:
node tools/plan-publish.js --execute
### Publish artifacts to registries
.PHONY: push
push:
@: $${I} $${DOCKER_REPO}
docker tag riju:$(I) $(DOCKER_REPO):$(I)
docker push $(DOCKER_REPO):$(I)
.PHONY: upload
upload:
@: $${L} $${T} $${S3_BUCKET}
aws s3 rm --recursive $(S3_HASH)
aws s3 cp $(BUILD)/$(DEB) $(S3_DEB)
hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null
.PHONY: publish
publish:
tools/publish.bash
### Miscellaneous
.PHONY: dockerignore
dockerignore:
echo "# This file is generated by 'make dockerignore', do not edit." > .dockerignore
cat .gitignore | sed 's#^#**/#' >> .dockerignore
.PHONY: env
env:
exec bash --rcfile <(cat ~/.bashrc - <<< 'PS1="[.env] $$PS1"')
tmux:
tmux attach || tmux new-session -s tmux

View File

@ -9,25 +9,6 @@ import { log } from "./util.js";
// populated at runtime and updated asynchronously.
export let langs = {};
// Correct whitespace problems in a language configuration,
// destructively. Return the fixed configuration.
//
// This basically removes leading and trailing whitespace from all
// values in the configuration recursively.
function fixupLangConfig(langConfig) {
if (typeof langConfig === "string") {
return langConfig.trim();
} else if (typeof langConfig === "object") {
for (const key in langConfig) {
if (langConfig.id === "whitespace" && key === "template") {
continue;
}
langConfig[key] = fixupLangConfig(langConfig[key]);
}
}
return langConfig;
}
// Read languages from JSON files in /opt/riju/langs, and update the
// global langs variable in this module. Never throw an error. If
// there is a problem then just leave the languages as they previously
@ -40,8 +21,8 @@ async function readLangsFromDisk() {
continue;
}
const id = path.parse(filename).name;
const langConfig = fixupLangConfig(
JSON.parse(await fs.readFile(`/opt/riju/langs/${filename}`, "utf-8"))
const langConfig = JSON.parse(
await fs.readFile(`/opt/riju/langs/${filename}`, "utf-8")
);
if (langConfig.id !== id) {
log.error(

View File

@ -1,16 +1,15 @@
import { spawn } from "child_process";
import fs from "fs";
import { promises as fs } from "fs";
import { v4 as getUUID } from "uuid";
import { langs } from "./langs";
import { MIN_UID, MAX_UID, borrowUser, ignoreUsers } from "./users";
import { MIN_UID, MAX_UID, borrowUser, ignoreUsers } from "./users.js";
import {
privilegedSetup,
privilegedSpawn,
privilegedTeardown,
run,
} from "./util";
} from "./util.js";
function die(msg) {
console.error(msg);
@ -22,20 +21,9 @@ function log(msg) {
}
async function main() {
const dirs = await new Promise((resolve, reject) =>
fs.readdir("/tmp/riju", (err, dirs) => (err ? reject(err) : resolve(dirs)))
);
const dirs = await fs.readdir("/tmp/riju");
const uids = (
await Promise.all(
dirs.map(
(dir) =>
new Promise((resolve, reject) =>
fs.stat(`/tmp/riju/${dir}`, (err, stat) =>
err ? reject(err) : resolve(stat.uid)
)
)
)
)
await Promise.all(dirs.map((dir) => fs.stat(`/tmp/riju/${dir}`)))
).filter((uid) => uid >= MIN_UID && uid < MAX_UID);
await ignoreUsers(uids, log);
const uuid = getUUID();

View File

@ -119,7 +119,7 @@ export function bash(cmdline) {
// single command (no shell logic).
cmdline = "exec " + cmdline;
}
return ["bash", "-c", cmdline];
return ["bash", "-c", `set -euo pipefail; ${cmdline}`];
}
export const log = {

View File

@ -22,8 +22,25 @@ deb https://deb.nodesource.com/${node_repo} ${ubuntu_name} main
deb https://dl.yarnpkg.com/debian/ stable main
EOF
packages="
fakeroot
git
less
make
man
nodejs
ripgrep
sudo
tmux
unzip
wget
yarn
"
apt-get update
apt-get install -y fakeroot less make man nodejs sudo unzip wget yarn
apt-get install -y $(sed 's/#.*//' <<< "${packages}")
rm -rf /var/lib/apt/lists/*

View File

@ -6,6 +6,5 @@ groupadd -g "$(stat -c %g "$PWD")" -o -p '!' -r riju
useradd -u "$(stat -c %u "$PWD")" -g "$(stat -c %g "$PWD")" -o -p '!' -m -N -l -s /usr/bin/bash -G sudo riju
runuser -u riju touch /home/riju/.sudo_as_admin_successful
runuser -u riju -- yarn install
exec runuser -u riju "$@"

View File

@ -61,7 +61,7 @@ deb [arch=amd64] https://dl.hhvm.com/ubuntu ${ubuntu_name} main
deb [arch=amd64] https://deb.nodesource.com/${node_repo} ${ubuntu_name} main
# R
deb [arch=amd64] https://cloud.r-project.org/bin/linux/ubuntu ${ubuntu_name}-${cran_repo} main
deb [arch=amd64] https://cloud.r-project.org/bin/linux/ubuntu ${ubuntu_name}-${cran_repo}/
# Yarn
deb [arch=amd64] https://dl.yarnpkg.com/debian/ stable main
@ -72,8 +72,24 @@ apt-get install -y dctrl-tools
libicu="$(grep-aptavail -wF Package 'libicu[0-9]+' -s Package -n | head -n1)"
apt-get update
apt-get install -y less clang jq "${libicu}" make man nodejs sudo tmux wget yarn
packages="
less
clang
jq
${libicu}
make
man
nodejs
sudo
tmux
vim
wget
yarn
"
apt-get install -y $(sed 's/#.*//' <<< "${packages}")
ver="$(latest_release watchexec/watchexec)"
wget "https://github.com/watchexec/watchexec/releases/download/${ver}/watchexec-${ver}-x86_64-unknown-linux-gnu.deb"

View File

@ -6,6 +6,5 @@ groupadd -g "$(stat -c %g "$PWD")" -o -p '!' -r riju
useradd -u "$(stat -c %u "$PWD")" -g "$(stat -c %g "$PWD")" -o -p '!' -m -N -l -s /usr/bin/bash -G sudo riju
runuser -u riju touch /home/riju/.sudo_as_admin_successful
runuser -u riju -- yarn install
exec runuser -u riju "$@"

View File

@ -3,6 +3,14 @@ aliases:
- "fishlang"
name: "><>"
install:
manual: |
wget https://gist.githubusercontent.com/anonymous/6392418/raw/fish.py
sed -i 's:^#!.*:#!/usr/bin/env python3:' fish.py
chmod +x fish.py
install -d "${pkg}/usr/local/bin"
mv fish.py "${pkg}/usr/local/bin/fish-lang"
info:
year: 2009
desc: "Stack-based, reflective, two-dimensional esoteric programming language"

View File

@ -4,6 +4,18 @@ name: "ABC"
install:
apt:
- libtinfo5:i386
manual: |
wget https://homepages.cwi.nl/~steven/abc/implementations/abc.tar.gz
install -d "${pkg}/opt/abc"
tar -xf abc.tar.gz -C "${pkg}/opt/abc" --strip-components=1
chmod +x "${pkg}/opt/abc/abc" "${pkg}/opt/abc/abckeys"
install -d "${pkg}/usr/local/bin"
tee "${pkg}/usr/local/bin/abc" >/dev/null <<"EOF"
#!/usr/bin/env bash
cd /opt/abc
exec ./abc "$@"
EOF
chmod +x "${pkg}/usr/local/bin/abc"
repl: |
abc

View File

@ -29,6 +29,13 @@ info:
install:
apt:
- gnat
manual: |
wget https://dl.bintray.com/reznikmm/ada-language-server/linux-latest.tar.gz
tar -xf linux-latest.tar.gz
install -d "${pkg}/usr/local/bin"
install -d "${pkg}/usr/lib/x86_64-linux-gnu"
mv linux/ada_language_server "${pkg}/usr/local/bin/ada_language_server"
mv linux/*.so* "${pkg}/usr/lib/x86_64-linux-gnu/"
main: "main.adb"
template: |

View File

@ -3,10 +3,36 @@ aliases:
- "an"
name: "Ante"
repl: |
ante
output: |
i32
install:
prepare:
apt:
- cargo
- clang
- cmake
- libssl-dev
- pkg-config
- python3-distutils
manual: |
export PATH="$HOME/.cargo/bin:$PATH"
cargo install llvmenv
llvmenv init
# If compiler is not explicitly set to LLVM, then we get an
# error: unrecognized command-line option -Wnewline-eof.
CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 10.0.0
llvmenv global 10.0.0
manual: |
git clone https://github.com/jfecher/ante.git
pushd ante
LLVM_SYS_100_PREFIX="$(llvmenv prefix)" cargo build --release
install -d "${pkg}/opt/ante"
install -d "${pkg}/usr/local/bin"
cp target/release/ante "${pkg}/usr/local/bin/"
cp -R stdlib "${pkg}/opt/ante/"
popd
setup: |
mkdir -p "$HOME/.config/ante"
cp -R /opt/ante/stdlib "$HOME/.config/ante/"
main: "main.an"
template: |
@ -15,4 +41,4 @@ template: |
compile: |
ante main.an
run: |
./main; ante
./main

View File

@ -1,6 +1,15 @@
id: "antecards"
name: "Ante (Cards)"
install:
apt:
- ruby
manual: |
wget https://github.com/michaeldv/ante/raw/master/ante.rb
chmod +x ante.rb
install -d "${pkg}/usr/local/bin"
mv ante.rb "${pkg}/usr/local/bin/ante-cards"
main: "main.ante"
template: |
9♦8♥J♦A♦2♣3♥7♠J♦A♦7♦J♦J♦A♦3♦J♦5♥6♦4♥J♥A♥6♠6♠J♥A♦8♦J♦A♦8♠J♦A♦3♦J♦A♦6♠J♦A♦8♠J♦A♥3♦2♠J♥A♥2♣6♠J♥

View File

@ -27,6 +27,10 @@ info:
install:
apt:
- libtinfo5
manual: |
wget "ftp://ftp.gnu.org/gnu/apl/$(curl -sS ftp://ftp.gnu.org/gnu/apl/ | grep -Eo 'apl_[-0-9.]+_amd64.deb$' | sort -rV | head -n1)" -O apl.deb
deb:
- apl.deb
repl: |
apl

View File

@ -19,7 +19,7 @@ info:
usage: personal
install:
build:
prepare:
apt:
- golang
apt:

View File

@ -6,7 +6,7 @@ aliases:
name: "Haskell"
install:
build:
prepare:
apt:
- cabal-install
apt:

View File

@ -5,7 +5,7 @@ aliases:
name: "Idris"
install:
build:
prepare:
apt:
- chezscheme
- gcc

View File

@ -5,7 +5,7 @@ aliases:
name: "Limbo"
install:
build:
prepare:
apt:
- gcc
- libc6-dev:i386

View File

@ -5,7 +5,7 @@ aliases:
name: "LOLCODE"
install:
build:
prepare:
apt:
- cmake

View File

@ -4,7 +4,7 @@ aliases:
name: "Oberon"
install:
build:
prepare:
apt:
- clang

View File

@ -2,7 +2,7 @@ id: "ook"
name: "Ook"
install:
build:
prepare:
apt:
- autoconf

View File

@ -41,14 +41,13 @@ install:
- python3
- python3-pip
- black
manual:
- |
install -d "${pkg}/opt/mspyls"
install -d "${pkg}/usr/local/bin"
wget "$(curl -fsSL "https://pvsc.blob.core.windows.net/python-language-server-stable?restype=container&comp=list&prefix=Python-Language-Server-linux-x64" | grep -Eo 'https://[^<]+\.nupkg' | tail -n1)"
unzip -d "${pkg}/opt/mspyls" Python-Language-Server-linux-x64.*.nupkg
chmod +x "${pkg}/opt/mspyls/Microsoft.Python.LanguageServer"
ln -s "/opt/mspyls/Microsoft.Python.LanguageServer" "${pkg}/usr/local/bin/Microsoft.Python.LanguageServer"
manual: |
install -d "${pkg}/opt/mspyls"
install -d "${pkg}/usr/local/bin"
wget "$(curl -fsSL "https://pvsc.blob.core.windows.net/python-language-server-stable?restype=container&comp=list&prefix=Python-Language-Server-linux-x64" | grep -Eo 'https://[^<]+\.nupkg' | tail -n1)"
unzip -d "${pkg}/opt/mspyls" Python-Language-Server-linux-x64.*.nupkg
chmod +x "${pkg}/opt/mspyls/Microsoft.Python.LanguageServer"
ln -s "/opt/mspyls/Microsoft.Python.LanguageServer" "${pkg}/usr/local/bin/Microsoft.Python.LanguageServer"
repl: |
python3 -u

View File

@ -6,7 +6,7 @@ aliases:
name: "SNOBOL"
install:
build:
prepare:
apt:
- m4

View File

@ -6,7 +6,7 @@ aliases:
name: "TeX"
install:
build:
prepare:
apt:
- luarocks
- $(grep-aptavail -eF Package "liblua[0-9.]+-dev" -a -XF Version "$(grep-aptavail -XF Provides lua -s Version -n | sort -Vr | head -n1)" -s Package -n | head -n1)

View File

@ -2,7 +2,7 @@ id: "zot"
name: "Zot"
install:
build:
prepare:
apt:
- qt5-qmake
- qtscript5-dev

View File

@ -7,7 +7,7 @@ aliases:
name: "Рапира"
install:
build:
prepare:
apt:
- clang

View File

@ -39,6 +39,25 @@ export async function getPackages() {
return packages;
}
// Correct whitespace problems in a language configuration,
// destructively. Return the fixed configuration.
//
// This basically removes leading and trailing whitespace from all
// values in the configuration recursively.
function fixupLangConfig(langConfig) {
if (typeof langConfig === "string") {
return langConfig.trim();
} else if (typeof langConfig === "object") {
for (const key in langConfig) {
if (langConfig.id === "whitespace" && key === "template") {
continue;
}
langConfig[key] = fixupLangConfig(langConfig[key]);
}
}
return langConfig;
}
// Read the YAML config file for the language with the given string ID
// and return it as an object.
export async function readLangConfig(lang) {
@ -50,5 +69,5 @@ export async function readLangConfig(lang) {
`lang config id ${langConfig.id} doesn't match expected ${lang}`
);
}
return langConfig;
return fixupLangConfig(langConfig);
}

View File

@ -14,33 +14,56 @@ function makeLangScript(langConfig) {
const {
id,
name,
install: { apt, pip, manual },
install: { prepare, apt, pip, manual, deb },
} = langConfig;
let parts = [];
parts.push(`\
#!/usr/bin/env bash
set -euxo pipefail`);
if (prepare) {
const { apt, manual } = prepare;
if (apt && apt.length > 0) {
parts.push(`\
export DEBIAN_FRONTEND=noninteractive
sudo apt-get update
sudo apt-get install -y ${apt.join(" ")}`);
}
if (manual) {
parts.push(manual);
}
}
if (manual) {
parts.push(manual);
}
if (deb) {
parts.push(
deb.map((deb) => `dpkg-deb --extract "${deb}" "\${pkg}"`).join("\n")
);
}
let depends = [];
if (apt) {
depends = depends.concat(apt);
}
if (deb) {
depends = depends.concat(
deb.map((fname) => `\$(dpkg-deb -f "${fname}" Depends)`)
);
}
parts.push(`depends=(${depends.map((dep) => `"${dep}"`).join(" ")})`);
let debianControlData = `\
Package: riju-lang-${id}
Version: \$(date +%s%3N)
Architecture: amd64
Maintainer: Radon Rosborough <radon.neon@gmail.com>
Description: The ${name} language packaged for Riju`;
if (apt.length > 0) {
debianControlData += `
Depends: ${apt.join(", ")}`;
}
debianControlData += `
Description: The ${name} language packaged for Riju
Depends: \$(IFS=,; echo "\${depends[*]}")
Riju-Script-Hash: \$(sha1sum "\$0" | awk '{ print \$1 }')`;
parts.push(`\
install -d "\${pkg}/DEBIAN"
cat <<EOF > "\${pkg}/DEBIAN/control"
${debianControlData}
EOF`);
for (const part of manual || []) {
parts.push(part);
}
return parts.join("\n\n");
}

View File

@ -19,7 +19,7 @@ function cleanup {
trap cleanup EXIT
make pull-base scripts
make pull-base all-scripts
node tools/plan-publish.js --execute --publish --show-all --omit-unneeded-downloads \
| tee "${tmpdir}/plan-publish.out"