From 6c678c73ce04849bfa588e366955674515ea3541 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Jun 2021 22:17:08 -0700 Subject: [PATCH] Start seriously screwing around with GBS.js --- Makefile | 36 +------ docker/base/install.bash | 71 +------------ docker/lang/install.bash | 9 ++ docker/packaging/install.bash | 20 +--- langs/ceylon.yaml | 8 ++ langs/crystal.yaml | 4 + langs/d.yaml | 7 +- langs/dart.yaml | 4 + langs/hack.yaml | 4 + langs/mongodb.yaml | 9 +- langs/qsharp.yaml | 9 +- langs/r.yaml | 4 + lib/yaml.js | 33 +++--- tools/depgraph.js | 1 - tools/generate-build-script.js | 184 ++++++++++++++++++++++++--------- tools/jsonschema.yaml | 29 ++++++ tools/make-foreach.js | 5 - 17 files changed, 235 insertions(+), 202 deletions(-) diff --git a/Makefile b/Makefile index 0ee66a2..0db51f8 100644 --- a/Makefile +++ b/Makefile @@ -96,28 +96,17 @@ repkg: script # L= T= : Build fresh .deb and install into live conta $(MAKE_QUIETLY) shell I=packaging CMD="make pkg L=$(L) T=$(T)" ctr="$$(docker container ls -f label="riju-install-target=yes" -l -q)"; test "$${ctr}" || (echo "no valid container is live"; exit 1); docker exec "$${ctr}" make install L=$(L) T=$(T) -## This is equivalent to 'make repkg T=lang', 'make repkg T=config'. -## For shared dependencies, use 'make repkg T=shared' directly. - -repkgs: # L= : Build and install fresh lang and config .debs - @: $${L} - node tools/make-foreach.js --types repkg L=$(L) - ### Build packaging scripts script: # L= T= : Generate a packaging script @: $${L} $${T} mkdir -p $(BUILD) node tools/generate-build-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash +ifeq ($(T),lang) + node tools/generate-build-script.js --lang $(L) --type install > $(BUILD)/install.bash +endif chmod +x $(BUILD)/build.bash -scripts: # L= : Generate both lang and config packaging scripts - @: $${L} - node tools/make-foreach.js --types script L=$(L) - -## This is equivalent to 'make script T=lang', 'make script T=config'. -## For shared dependencies, use 'make script T=shared' directly. - all-scripts: # Generate packaging scripts for all languages node tools/write-all-build-scripts.js @@ -146,20 +135,9 @@ pkg-deb: # L= T= [Z=gzip|xz] : Build .deb from packaging environment @: $${L} $${T} fakeroot dpkg-deb --build -Z$(Z) $(BUILD)/pkg $(BUILD)/$(DEB) -## This is equivalent to the sequence 'script', 'pkg-clean', 'pkg-build', 'pkg-deb'. +## This is equivalent to the sequence 'pkg-clean', 'pkg-build', 'pkg-deb'. -pkg: script pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb - -## This is equivalent to 'make pkg T=lang', 'make pkg T=config'. For -## shared dependencies, use 'make pkg T=shared' directly. -# -## Z is the compression type to use; defaults to none. Higher -## compression levels (gzip is moderate, xz is high) take much longer -## but produce much smaller packages. - -pkgs: # L= [Z=gzip|xz] : Build both lang and config .debs - @: $${L} - node tools/make-foreach.js --types pkg L=$(L) +pkg: pkg-clean pkg-build pkg-deb # L= T= [Z=gzip|xz] : Build fresh .deb ### Install packages @@ -168,10 +146,6 @@ install: # L= T= : Install built .deb if [[ -z "$$(ls -A /var/lib/apt/lists)" ]]; then sudo apt update; fi DEBIAN_FRONTEND=noninteractive sudo -E apt reinstall -y ./$(BUILD)/$(DEB) -installs: # L= : Install both lang and config .debs - @: $${L} - node tools/make-foreach.js --types install L=$(L) - ### Build and run application code frontend: # Compile frontend assets for production diff --git a/docker/base/install.bash b/docker/base/install.bash index ddf82b3..5bbc922 100755 --- a/docker/base/install.bash +++ b/docker/base/install.bash @@ -11,99 +11,30 @@ pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive -dpkg --add-architecture i386 - apt-get update +apt-get dist-upgrade -y (yes || true) | unminimize apt-get install -y curl gnupg lsb-release wget -# Ceylon -wget https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem -O /usr/local/share/ca-certificates/DigiCertTLSRSASHA2562020CA1.crt - -# D -wget https://letsencrypt.org/certs/lets-encrypt-r3.pem -O /usr/local/share/ca-certificates/lets-encrypt-r3.crt - -update-ca-certificates - -ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" -cran_repo="$(curl -fsSL https://cran.r-project.org/bin/linux/ubuntu/ | grep -Eo 'cran[0-9]+' | head -n1)" node_repo="$(curl -fsSL https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" -# .NET -wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" -apt-get install ./packages-microsoft-prod.deb - -# Ceylon -curl -fsSL https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key | apt-key add - - -# Crystal -curl -fsSL https://keybase.io/crystal/pgp_keys.asc | apt-key add - - -# Dart -curl -fsSL https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - - -# Hack -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B4112585D386EB94 - -# MongoDB -curl -fsSL https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - - # Node.js curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - -# R -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 - # Yarn curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null < "install-lang-${LANG}.bash" riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" ( @@ -21,6 +22,7 @@ riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb" (grep -Eo 'riju-shared-[^, ]+' || true) | sed 's/riju-shared-//' ) | while read name; do + riju-curl "build/shared/${name}/install.bash" > "install-shared-${name}.bash" riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb" done @@ -28,12 +30,19 @@ if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then apt-get update fi +if compgen -G "./install-shared-*.bash"; then + for file in ./install-shared-*.bash; do + "${file}" + done +fi + if compgen -G "./riju-shared-*.deb"; then for file in ./riju-shared-*.deb; do apt-get install -y "${file}" done fi +"./install-lang-${LANG}.bash" apt-get install -y "./riju-lang-${LANG}.deb" popd diff --git a/docker/packaging/install.bash b/docker/packaging/install.bash index 1b08ef3..b3a0aa7 100755 --- a/docker/packaging/install.bash +++ b/docker/packaging/install.bash @@ -10,42 +10,24 @@ pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive -dpkg --add-architecture i386 - apt-get update +apt-get dist-upgrade -y (yes || true) | unminimize apt-get install -y curl gnupg lsb-release wget -wget https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem -O /usr/local/share/ca-certificates/DigiCertTLSRSASHA2562020CA1.crt -wget https://letsencrypt.org/certs/lets-encrypt-r3.pem -O /usr/local/share/ca-certificates/lets-encrypt-r3.crt - -update-ca-certificates - -ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" -wget "https://packages.microsoft.com/config/ubuntu/${ubuntu_ver}/packages-microsoft-prod.deb" -apt-get install ./packages-microsoft-prod.deb - -curl -fsSL https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key | apt-key add - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null </dev/null < pkg.includes("$")).length > 0)) ) { parts.push(`\ -export DEBIAN_FRONTEND=noninteractive -sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); +export DEBIAN_FRONTEND=noninteractive`); + if ( + install.prepare && + ((install.prepare.manual && + install.prepare.manual.includes("apt-get") && + install.prepare.manual.includes(":i386")) || + (install.prepare.apt && + install.prepare.apt.filter((pkg) => pkg.includes(":i386")).length > + 0)) + ) { + parts.push(`\ +dpkg --add-architecture i386`); + } + parts.push(`\ + sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); } if (install) { const { @@ -49,23 +64,67 @@ sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); deb, } = install; if (prepare) { - const { apt, npm, opam, manual } = prepare; + const { + preface, + cert, + aptKey, + aptRepo, + apt, + npm, + opam, + manual, + } = prepare; + if (preface) { + prefaceParts.push(preface); + } + if (cert && cert.length > 0) { + prefaceParts.push( + cert + .map( + (url, idx) => + `sudo wget "${url}" -O /usr/local/share/ca-certificates/riju-${id}-${idx}.crt` + ) + .join("\n") + ); + prefaceParts.push(`sudo update-ca-certificates`); + } + if (aptKey && aptKey.length > 0) { + prefaceParts.push( + aptKey + .map((src) => { + if (src.startsWith("http://") || src.startsWith("https://")) { + return `curl -fsSL "${src}" | sudo apt-key add -`; + } else if (/^[0-9A-F]+$/.match(src)) { + return `sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${src}"`; + } else { + throw new Error(`unknown aptKey format: ${src}`); + } + }) + .join("\n") + ); + } + if (aptRepo && aptRepo.length > 0) { + prefaceParts.push(`sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null < 0) { - parts.push(`\ + needsAptGetUpdate = true; + prefaceParts.push(`\ sudo --preserve-env=DEBIAN_FRONTEND apt-get install -y ${apt.join(" ")}`); } if (npm && npm.length > 0) { - parts.push(`\ + prefaceParts.push(`\ sudo npm install -g ${npm.join(" ")}`); } if (opam && opam.length > 0) { - parts.push(`\ + prefaceParts.push(`\ sudo opam init -n --disable-sandboxing --root /opt/opam sudo opam install "${opam.join(" ")}" -y --root /opt/opam sudo ln -s /opt/opam/default/bin/* /usr/local/bin/`); } if (manual) { - parts.push(manual); + prefaceParts.push(manual); } } if (npm && npm.length > 0) { @@ -215,6 +274,9 @@ chmod +x "${path}"`); } } if (manual) { + if (manual.includes("apt-get")) { + needsAptGetUpdate = true; + } parts.push(manual); } if (deb) { @@ -223,6 +285,9 @@ chmod +x "${path}"`); ); } if (apt) { + if (apt.filter((pkg) => pkg.includes("$")).length > 0) { + needsAptGetUpdate = true; + } depends = depends.concat(apt); } if (dependsCfg.unpin) { @@ -237,6 +302,13 @@ chmod +x "${path}"`); ); } } + if (needsAptGetUpdate) { + prefaceParts.unshift(`\ +export DEBIAN_FRONTEND=noninteractive`); + prefaceParts.push(`\ +sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); + } + parts = prefaceParts.concat(parts); parts.push(`depends=(${depends.map((dep) => `"${dep}"`).join(" ")})`); let stripDependsFilter = ""; const stripDepends = (dependsCfg.strip || []).concat(dependsCfg.unpin || []); @@ -280,40 +352,6 @@ set -euxo pipefail`); return parts.join("\n\n"); } -// Given a language config object, return the text of a Bash script -// that will build the (unpacked) riju-config-foo Debian package into -// ${pkg} when run in an appropriate environment. This is a package -// that will install configuration files and/or small scripts that -// encode the language configuration so that Riju can operate on any -// installed languages without knowing their configuration in advance. -function makeConfigScript(langConfig) { - const { id, name } = langConfig; - let parts = []; - parts.push(`\ -#!/usr/bin/env bash - -set -euxo pipefail`); - let debianControlData = `\ -Package: riju-config-${id} -Version: \$(date +%s%3N) -Architecture: all -Maintainer: Radon Rosborough -Description: Riju configuration for the ${name} language -Depends: riju-lang-${id} -Riju-Script-Hash: \$(sha1sum "$0" | awk '{ print $1 }')`; - parts.push(`\ -install -d "\${pkg}/DEBIAN" -cat < "\${pkg}/DEBIAN/control" -${debianControlData} -EOF`); - parts.push(`\ -install -d "\${pkg}/opt/riju/langs" -cat <<"EOF" > "\${pkg}/opt/riju/langs/${id}.json" -${JSON.stringify(langConfig, null, 2)} -EOF`); - return parts.join("\n\n"); -} - // Given a language config object, return the text of a Bash script // that will build the (unpacked) riju-shared-foo Debian package into // ${pkg} when run in an appropriate environment. This is a package @@ -323,20 +361,66 @@ function makeSharedScript(langConfig) { return makeLangScript(langConfig, true); } +// Given a language ID, return the text of a Bash script that will do +// any necessary setup before the language package is installed (along +// with its shared dependencies, if any). +function makeInstallScript(lang) { + let parts = []; + if (install) { + const { apt, cert, aptKey, aptRepo } = install; + if (apt && apt.filter((pkg) => pkg.includes(":i386")).length > 0) { + parts.push(`\ +dpkg --add-architecture i386`); + } + if (cert && cert.length > 0) { + parts.push( + cert + .map( + (url, idx) => + `sudo wget "${url}" -O /usr/local/share/ca-certificates/riju-${id}-${idx}.crt` + ) + .join("\n") + ); + parts.push(`sudo update-ca-certificates`); + } + if (aptKey && aptKey.length > 0) { + parts.push( + aptKey + .map((src) => { + if (src.startsWith("http://") || src.startsWith("https://")) { + return `curl -fsSL "${src}" | sudo apt-key add -`; + } else if (/^[0-9A-F]+$/.match(src)) { + return `sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${src}"`; + } else { + throw new Error(`unknown aptKey format: ${src}`); + } + }) + .join("\n") + ); + } + if (aptRepo && aptRepo.length > 0) { + parts.push(`sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null < makeLangScript(await readLangConfig(lang)), + shared: async () => makeSharedScript(await readSharedDepConfig(lang)), + install: async () => await makeInstallScript(lang), }[type]; if (!scriptMaker) { throw new Error(`unsupported script type ${type}`); } - return scriptMaker( - type === "shared" - ? await readSharedDepConfig(lang) - : await readLangConfig(lang) - ); + return scriptMaker(); } // Parse command-line arguments, run main functionality, and exit. @@ -346,7 +430,7 @@ async function main() { .requiredOption("--lang ", "language ID") .requiredOption( "--type ", - "package category (lang, config, shared)" + "package category (lang, shared, install)" ); program.parse(process.argv); console.log(await generateBuildScript(program.opts())); diff --git a/tools/jsonschema.yaml b/tools/jsonschema.yaml index dac9c91..edc3255 100644 --- a/tools/jsonschema.yaml +++ b/tools/jsonschema.yaml @@ -511,6 +511,31 @@ properties: type: object additionalProperties: false properties: + preface: &preface + type: string + minLength: 1 + cert: &cert + type: array + items: + type: string + pattern: "^https?://" + examples: + - "https://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1.crt.pem" + aptKey: &aptKey + type: array + items: + type: string + pattern: "^https?://|^[0-9A-F]+$" + examples: + - "https://downloads.ceylon-lang.org/apt/ceylon-debian-repo.gpg.key" + - "B4112585D386EB94" + aptRepo: &aptRepo + type: array + items: + type: string + pattern: "^https?://" + examples: + - "https://downloads.ceylon-lang.org/apt/ unstable main" apt: type: array items: @@ -529,6 +554,10 @@ properties: manual: type: string minLength: 1 + preface: *preface + cert: *cert + aptKey: *aptKey + aptRepo: *aptRepo apt: type: array items: diff --git a/tools/make-foreach.js b/tools/make-foreach.js index 56fc1e5..143cadd 100644 --- a/tools/make-foreach.js +++ b/tools/make-foreach.js @@ -20,11 +20,6 @@ async function main() { ); } break; - case "--types": - for (const type of ["lang", "config"]) { - await runCommand(`MAKELEVEL= make ${targets.join(" ")} T=${type}`); - } - break; default: console.error(`make-foreach.js: unknown selector: ${selector}`); process.exit(1);