Rearrange again, enable Docker-in-Docker-in-Docker

.dockerignore

@@ -3,4 +3,5 @@
 **/.env
 **/.terraform
 **/build
+**/composite
 **/node_modules

.gitignore

@@ -2,4 +2,5 @@
 .env
 .terraform
 build
+composite
 node_modules

Makefile

@@ -12,11 +12,6 @@ S3_DEBS := s3://$(S3_BUCKET_BASE)-debs
 S3_DEB := $(S3_DEBS)/debs/$(DEB)
 S3_HASH := $(S3_DEBS)/hashes/riju-$(T)-$(L)
 
-SHELL_ARGS :=
-ifeq ($(I),admin)
-SHELL_ARGS := -v $(HOME)/.aws:/var/riju/.aws:ro -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY
-endif
-
 .PHONY: help
 help:
 	@echo "usage:"
@@ -31,13 +26,17 @@ help:
 .PHONY: image
 image:
 	@: $${I}
+ifeq ($(I),composite)
+	node src/build-composite-image.js
+else
 	docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --pull
+endif
 
 .PHONY: script
 script:
 	@: $${L} $${T}
 	mkdir -p $(BUILD)
-	node src/packager/make-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash
+	node src/make-script.js --lang $(L) --type $(T) > $(BUILD)/build.bash
 	chmod +x $(BUILD)/build.bash
 
 .PHONY: pkg
@@ -50,10 +49,16 @@ pkg:
 
 ### Run things inside Docker
 
+VOLUME_MOUNT ?= $(PWD)
+
 .PHONY: shell
 shell:
 	@: $${I}
-	docker run -it --rm -v $(PWD):/src $(SHELL_ARGS) riju:$(I)
+ifeq ($(I),admin)
+	docker run -it --rm -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws:ro -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e VOLUME_MOUNT=$(VOLUME_MOUNT) riju:$(I)
+else
+	docker run -it --rm -v $(VOLUME_MOUNT):/src riju:$(I)
+endif
 
 .PHONY: install
 install:

docker/admin/install.bash

@@ -11,8 +11,9 @@ apt-get update
 
 apt-get install -y curl gnupg lsb-release
 
-curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
-curl -sSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
+curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
 
 ubuntu_ver="$(lsb_release -rs)"
 ubuntu_name="$(lsb_release -cs)"
@@ -20,12 +21,13 @@ ubuntu_name="$(lsb_release -cs)"
 node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)"
 
 tee -a /etc/apt/sources.list.d/custom.list >/dev/null <<EOF
-deb https://deb.nodesource.com/${node_repo} ${ubuntu_name} main
-deb https://dl.yarnpkg.com/debian/ stable main
+deb [arch=amd64] https://deb.nodesource.com/${node_repo} ${ubuntu_name} main
+deb [arch=amd64] https://dl.yarnpkg.com/debian/ stable main
+deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable
 EOF
 
 apt-get update
-apt-get install -y less make man nodejs sudo unzip wget yarn
+apt-get install -y docker-ce-cli less make man nodejs sudo unzip wget yarn
 
 wget https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip
 unzip awscli.zip

docker/admin/pid1.bash

@@ -7,5 +7,6 @@ useradd -u "$(stat -c %u "$PWD")" -g "$(stat -c %g "$PWD")" -o -m -N -l -s /usr/
 
 runuser -u riju-admin -- touch /home/riju-admin/.sudo_as_admin_successful
 runuser -u riju-admin -- ln -sT /var/riju/.aws /home/riju-admin/.aws
+runuser -u riju-admin -- yarn install
 
 exec runuser -u riju-admin "$@"

langs/python.yaml

@@ -15,7 +15,7 @@ install:
     - |
       install -d "${pkg}/opt/mspyls"
       install -d "${pkg}/usr/local/bin"
-      wget "$(curl -sSL "https://pvsc.blob.core.windows.net/python-language-server-stable?restype=container&comp=list&prefix=Python-Language-Server-linux-x64" | grep -Eo 'https://[^<]+\.nupkg' | tail -n1)"
+      wget "$(curl -fsSL "https://pvsc.blob.core.windows.net/python-language-server-stable?restype=container&comp=list&prefix=Python-Language-Server-linux-x64" | grep -Eo 'https://[^<]+\.nupkg' | tail -n1)"
       unzip -d "${pkg}/opt/mspyls" Python-Language-Server-linux-x64.*.nupkg
       chmod +x "${pkg}/opt/mspyls/Microsoft.Python.LanguageServer"
       ln -s "/opt/mspyls/Microsoft.Python.LanguageServer" "${pkg}/usr/local/bin/Microsoft.Python.LanguageServer"

src/build-composite-image.js

@@ -0,0 +1,15 @@
+import { promises as fs } from "fs";
+
+import { getLangs } from "./config.js";
+
+// Parse command-line arguments, run main functionality, and exit.
+async function main() {
+  await fs.mkdir("docker/composite", { recursive: true });
+  const langs = await getLangs();
+  // await fs.writeFile("docker/composite/Dockerfile", )
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

src/config.js

@@ -1,5 +1,5 @@
-import _fs from "fs";
-const fs = _fs.promises;
+import { promises as fs } from "fs";
+import path from "path";
 
 import YAML from "yaml";
 
@@ -12,6 +12,14 @@ import YAML from "yaml";
 //   directory also starts out empty
 // * we are using bash with 'set -euxo pipefail'
 
+// Return a list of the IDs of all the configured languages. Each such
+// ID can be passed to readLangConfig.
+export async function getLangs() {
+  return (await fs.readdir("langs"))
+    .filter((lang) => lang.endsWith(".yaml"))
+    .map((lang) => path.parse(lang).name);
+}
+
 // Read the YAML config file for the language with the given string ID
 // and return it as an object.
 export async function readLangConfig(lang) {

src/generate-build-script.js

@@ -3,7 +3,7 @@ import process from "process";
 import { Command } from "commander";
 import YAML from "yaml";
 
-import { readLangConfig } from "../config.js";
+import { readLangConfig } from "./config.js";
 
 // Given a language config object, return the text of a Bash script
 // that will build the (unpacked) riju-lang-foo Debian package into

src/packager/build.js

@@ -1,125 +0,0 @@
-const child_process = require("child_process");
-const fs = require("fs").promises;
-const path = require("path");
-const process = require("process");
-
-const YAML = require("yaml");
-
-// The build scripts in the language configs assume a specific build
-// environment, with these parameters:
-//
-// * the working directory starts out empty
-// * the ${pkg} environment variable has been set to an absolute path
-//   to the directory where the package should be built
-// * we are using bash with 'set -euo pipefail'
-
-// Used to log all progress messages. Not sure what this should do
-// quite yet.
-function log(message) {
-  console.error(message ? message.trimEnd() : "");
-}
-
-// Given a shell command as a string, execute it with Bash.
-async function runCommand(cmd) {
-  log(`$ ${cmd}`);
-  return new Promise((resolve, reject) => {
-    const proc = child_process.spawn(
-      "bash",
-      ["-c", `set -euo pipefail; ${cmd}`],
-      {
-        stdio: "inherit",
-      }
-    );
-    proc.on("error", reject);
-    proc.on("close", (code) => {
-      if (code === 0) {
-        resolve();
-      } else {
-        reject(new Error(`command exited with code ${code}`));
-      }
-    });
-  });
-}
-
-// Given a language config object, assuming that the environment has
-// already been set up properly (see top of this file for details),
-// run all the commands needed to build a package, and put the
-// resulting deb file into debPath.
-async function buildPackage(langConfig, debPath) {
-  const {
-    id,
-    name,
-    install: { apt, pip, manual },
-  } = langConfig;
-  const timestamp = new Date().getTime();
-  const pkgdir = process.env.pkg;
-  log();
-  log(`Building package riju-lang-${id}...`);
-  let debianControlData = `\
-Package: riju-lang-${id}
-Version: ${timestamp}
-Architecture: amd64
-Maintainer: Radon Rosborough <radon.neon@gmail.com>
-Description: The ${name} language packaged for Riju
-`;
-  if (apt.length > 0) {
-    debianControlData += `\
-Depends: ${apt.join(", ")}
-`;
-  }
-  log("Writing Debian control file:");
-  log(debianControlData.replaceAll(/^/gm, "  "));
-  await fs.mkdir(`${pkgdir}/DEBIAN`);
-  await fs.writeFile(`${pkgdir}/DEBIAN/control`, debianControlData);
-  await runCommand(`fakeroot dpkg-deb --build ${pkgdir} ${debPath}`);
-  log(`Finished building package riju-lang-${id}.`);
-  log();
-}
-
-// Create a temporary directory and call the given sync or async
-// function with its path as a string. Once the function returns, make
-// sure the directory and its contents are deleted.
-async function withTempDir(cb) {
-  return await tmp.withDir(
-    async (o) => {
-      await cb(o.path);
-    },
-    {
-      unsafeCleanup: true,
-    }
-  );
-}
-
-// Parse command line and run main functionality. This changes the
-// process environment destructively.
-async function main() {
-  const args = process.argv.slice(2);
-  if (args.length !== 1) {
-    console.error("usage: build.js LANG");
-    process.exit(1);
-  }
-  const [lang] = args;
-  const cwd = process.cwd();
-  const srcdir = `${cwd}/work/${lang}/src`;
-  const pkgdir = `${cwd}/work/${lang}/pkg`;
-  const debPath = `${cwd}/debs/riju-lang-${lang}.deb`;
-  const langConfig = await readLangConfig(lang);
-  log(`Source directory: ${srcdir}`);
-  log(`Package directory: ${pkgdir}`);
-  log(`Will write .deb file to: ${debPath}`);
-  await fs.rmdir(srcdir, { recursive: true });
-  await fs.rmdir(pkgdir, { recursive: true });
-  await fs.rm(debPath, { force: true });
-  await fs.mkdir(srcdir, { recursive: true });
-  await fs.mkdir(pkgdir, { recursive: true });
-  await fs.mkdir(path.dirname(debPath), { recursive: true });
-  process.chdir(srcdir);
-  process.env.pkg = pkgdir;
-  await buildPackage(langConfig, debPath);
-}
-
-main().catch((err) => {
-  console.error();
-  console.error(err);
-  process.exit(1);
-});

src/packager/debug.js

@@ -1,28 +0,0 @@
-const fs = require("fs").promises;
-const process = require("process");
-
-const YAML = require("yaml");
-
-async function main() {
-  const args = process.argv.slice(2);
-
-  if (args.length !== 1) {
-    console.error("usage: debug.js LANG");
-    process.exit(1);
-  }
-
-  const [lang] = args;
-
-  console.log(
-    JSON.stringify(
-      YAML.parse(await fs.readFile(`langs/${lang}.yaml`, "utf-8")),
-      null,
-      2
-    )
-  );
-}
-
-main().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});