Build per-language Docker images
This commit is contained in:
Radon Rosborough
parent
40c8c08bed
commit
feb5aad8f0
24
Makefile
24
Makefile
|
|
@ -36,10 +36,11 @@ endif
|
|||
## Pass NC=1 to disable the Docker cache. Base images are not pulled;
|
||||
## see 'make pull-base' for that.
|
||||
|
||||
image: # I=<image> [NC=1] : Build a Docker image
|
||||
image: # I=<image> [L=<lang>] [NC=1] : Build a Docker image
|
||||
@: $${I}
|
||||
ifeq ($(I),composite)
|
||||
node tools/build-composite-image.js
|
||||
ifeq ($(I),lang)
|
||||
@: $${L}
|
||||
node tools/build-lang-image.js --lang $(L)
|
||||
else ifneq (,$(filter $(I),admin ci))
|
||||
docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
|
||||
else
|
||||
|
|
@ -59,14 +60,23 @@ endif
|
|||
|
||||
SHELL_ENV := -e Z -e CI -e TEST_PATIENCE -e TEST_CONCURRENCY
|
||||
|
||||
shell: # I=<shell> [E=1] [P1|P2=<port>] : Launch Docker image with shell
|
||||
ifeq ($(I),lang)
|
||||
LANG_TAG := lang-$(L)
|
||||
else
|
||||
LANG_TAG := $(I)
|
||||
endif
|
||||
|
||||
shell: # I=<shell> [L=<lang>] [E=1] [P1|P2=<port>] : Launch Docker image with shell
|
||||
@: $${I}
|
||||
ifneq (,$(filter $(I),admin ci))
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD)
|
||||
else ifneq (,$(filter $(I),compile app))
|
||||
else ifeq ($(I),app)
|
||||
docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
|
||||
else ifneq (,$(filter $(I),runtime composite))
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
|
||||
else ifneq (,$(filter $(I),runtime lang))
|
||||
ifeq ($(I),lang)
|
||||
@: $${L}
|
||||
endif
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD)
|
||||
else
|
||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
|
||||
endif
|
||||
|
|
|
|||
|
|
@ -1,12 +1,37 @@
|
|||
FROM riju:compile AS compile
|
||||
FROM riju:composite
|
||||
FROM ubuntu:rolling AS build
|
||||
|
||||
COPY docker/app/install-build.bash /tmp/
|
||||
RUN /tmp/install-build.bash
|
||||
|
||||
WORKDIR /src
|
||||
COPY Makefile ./
|
||||
|
||||
COPY system ./system/
|
||||
RUN make system
|
||||
|
||||
COPY package.json yarn.lock ./
|
||||
RUN yarn install
|
||||
|
||||
COPY webpack.config.cjs ./
|
||||
COPY frontend/src ./frontend/src/
|
||||
RUN make frontend
|
||||
|
||||
COPY frontend/pages ./frontend/pages/
|
||||
COPY frontend/styles ./frontend/styles/
|
||||
COPY backend ./backend/
|
||||
|
||||
FROM ubuntu:rolling
|
||||
|
||||
COPY docker/app/install.bash /tmp/
|
||||
RUN /tmp/install.bash
|
||||
|
||||
COPY docker/shared/my_init /usr/local/sbin/
|
||||
ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"]
|
||||
|
||||
RUN useradd -p '!' -m -l -s /usr/bin/bash riju
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY --chown=riju:riju --from=compile /src ./
|
||||
COPY --chown=riju:riju --from=build /src ./
|
||||
RUN chown root:riju system/out/*-privileged && chmod a=,g=rx,u=rwxs system/out/*-privileged
|
||||
|
||||
USER riju
|
||||
|
|
|
|||
|
|
@ -24,3 +24,7 @@ EOF
|
|||
|
||||
apt-get update
|
||||
apt-get install -y clang g++ make nodejs sudo yarn
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
rm "$0"
|
||||
|
|
@ -6,6 +6,7 @@ export DEBIAN_FRONTEND=noninteractive
|
|||
|
||||
apt-get update
|
||||
apt-get dist-upgrade -y
|
||||
|
||||
apt-get install -y curl gnupg lsb-release
|
||||
|
||||
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
FROM ubuntu:rolling
|
||||
|
||||
COPY docker/compile/install.bash /tmp/
|
||||
RUN /tmp/install.bash
|
||||
|
||||
WORKDIR /src
|
||||
COPY Makefile ./
|
||||
|
||||
COPY system ./system/
|
||||
RUN make system
|
||||
|
||||
COPY package.json yarn.lock ./
|
||||
RUN yarn install
|
||||
|
||||
COPY webpack.config.cjs ./
|
||||
COPY frontend/src ./frontend/src/
|
||||
RUN make frontend
|
||||
|
||||
COPY frontend/pages ./frontend/pages/
|
||||
COPY frontend/styles ./frontend/styles/
|
||||
COPY backend ./backend/
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
FROM riju:runtime
|
||||
|
||||
COPY docker/composite/install.bash /tmp/
|
||||
|
||||
# The number of commands here must match NUM_SHARDS in
|
||||
# build-composite-image.js.
|
||||
RUN /tmp/install.bash 0
|
||||
RUN /tmp/install.bash 1
|
||||
RUN /tmp/install.bash 2
|
||||
RUN /tmp/install.bash 3
|
||||
RUN /tmp/install.bash 4
|
||||
RUN /tmp/install.bash 5
|
||||
RUN /tmp/install.bash 6
|
||||
RUN /tmp/install.bash 7
|
||||
RUN /tmp/install.bash 8
|
||||
RUN /tmp/install.bash 9
|
||||
|
||||
RUN rm /tmp/install.bash
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
shard="$1"
|
||||
|
||||
function riju-curl {
|
||||
curl -fsSL "localhost:8487$1"
|
||||
}
|
||||
|
||||
function riju-apt-install {
|
||||
riju-curl "$1" > "$(basename "$1")"
|
||||
apt-get install -y "./$(basename "$1")"
|
||||
}
|
||||
|
||||
pushd /tmp
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
apt-get update
|
||||
|
||||
riju-curl "/shard/${shard}" | while read path; do
|
||||
riju-apt-install "/fs/${path}"
|
||||
done
|
||||
|
||||
rm -rf *.deb
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
popd
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
FROM riju:runtime
|
||||
|
||||
ARG LANG
|
||||
|
||||
COPY docker/lang/install.bash /tmp/
|
||||
RUN /tmp/install.bash
|
||||
|
||||
ENTRYPOINT ["/usr/local/sbin/my_init", "--quiet", "--"]
|
||||
RUN rm /usr/local/sbin/pid1.bash
|
||||
|
||||
RUN useradd -p '!' -m -l -s /usr/bin/bash riju
|
||||
WORKDIR /home/riju/src
|
||||
USER riju
|
||||
CMD ["bash"]
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${LANG}"
|
||||
|
||||
mkdir /tmp/riju-work
|
||||
pushd /tmp/riju-work
|
||||
|
||||
function riju-curl {
|
||||
echo >&2 "fetching ./$1"
|
||||
curl -fsSL "localhost:8487/fs/$1"
|
||||
}
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
riju-curl "build/lang/${LANG}/riju-lang-${LANG}.deb" > "riju-lang-${LANG}.deb"
|
||||
|
||||
(
|
||||
dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends |
|
||||
(grep -Eo 'riju-shared-[^, ]+' || true) |
|
||||
sed 's/riju-shared-//'
|
||||
) | while read name; do
|
||||
riju-curl "build/shared/${name}/riju-shared-${name}.deb" > "riju-shared-${name}.deb"
|
||||
done
|
||||
|
||||
if dpkg-deb -f "riju-lang-${LANG}.deb" -f Depends | grep .; then
|
||||
apt-get update
|
||||
fi
|
||||
|
||||
for file in ./riju-shared-*.deb; do
|
||||
apt-get install -y "${file}"
|
||||
done
|
||||
|
||||
apt-get install -y "./riju-lang-${LANG}.deb"
|
||||
|
||||
popd
|
||||
rm -rf /tmp/riju-work
|
||||
|
||||
rm "$0"
|
||||
|
|
@ -90,14 +90,14 @@ EOF
|
|||
# Unfortunately, the Microsoft repo includes a duplicate version of
|
||||
# the libodbc1 package whose version is not in sync with the one
|
||||
# shipped by the corresponding release of Ubuntu. If this one happens
|
||||
# to be newer, then it'll cause a horrifyingly difficult to diagnose
|
||||
# error later on while building the composite image because there's a
|
||||
# conflict between the default-available versions of libodbc1 and
|
||||
# libodbc1:i386, which surfaces as an inability to install
|
||||
# dependencies for Erlang. Thanks Microsoft. Please don't. Anyway,
|
||||
# solution is to pin this repository at a lower priority than the
|
||||
# Ubuntu standard packages, so the correct version of libodbc1 gets
|
||||
# installed by default.
|
||||
# to be newer, then it can cause horrifyingly difficult to diagnose
|
||||
# errors later on because there's a conflict between the
|
||||
# default-available versions of libodbc1 and libodbc1:i386, which has
|
||||
# in the past surfaced as an inability to install dependencies for
|
||||
# Erlang. Thanks Microsoft. Please don't. Anyway, solution is to pin
|
||||
# this repository at a lower priority than the Ubuntu standard
|
||||
# packages, so the correct version of libodbc1 gets installed by
|
||||
# default.
|
||||
tee -a /etc/apt/preferences.d/riju >/dev/null <<EOF
|
||||
Package: *
|
||||
Pin: origin packages.microsoft.com
|
||||
|
|
|
|||
|
|
@ -1,97 +0,0 @@
|
|||
import { promises as fs } from "fs";
|
||||
import http from "http";
|
||||
|
||||
import express from "express";
|
||||
|
||||
import { getLangs, getPackages, getSharedDeps } from "./config.js";
|
||||
import { getLocalImageLabel } from "./docker-util.js";
|
||||
import { hashDockerfile } from "./hash-dockerfile.js";
|
||||
import { runCommand } from "./util.js";
|
||||
|
||||
// Number of package installation layers in the composite Docker
|
||||
// image. This needs to match the number of installation RUN commands
|
||||
// in the composite Dockerfile.
|
||||
const NUM_SHARDS = 10;
|
||||
|
||||
// Get a Node.js http server object that will serve information and
|
||||
// files for packages that should be installed into the composite
|
||||
// Docker image.
|
||||
function getServer({ shards }) {
|
||||
const app = express();
|
||||
app.get("/shard/:shard", (req, res) => {
|
||||
res.send(
|
||||
shards[parseInt(req.params.shard)]
|
||||
.map(({ debPath }) => debPath + "\n")
|
||||
.join("")
|
||||
);
|
||||
});
|
||||
app.use("/fs", express.static("."));
|
||||
return http.createServer(app);
|
||||
}
|
||||
|
||||
// Given a list of the packages to be built, split them into shards.
|
||||
// Return a list of shards. Each shard is a list of the package
|
||||
// objects, such that there are NUM_SHARDS shards. Traversing each
|
||||
// shard in order will return the packages in the same order as the
|
||||
// original list.
|
||||
//
|
||||
// Currently this uses an extremely simple algorithm, but that might
|
||||
// be improved in the future.
|
||||
function getShards(pkgs) {
|
||||
const shards = [];
|
||||
for (let i = 0; i < NUM_SHARDS; ++i) {
|
||||
shards.push([]);
|
||||
}
|
||||
const shardSize = Math.ceil(pkgs.length / NUM_SHARDS);
|
||||
for (let i = 0; i < pkgs.length; ++i) {
|
||||
shards[Math.floor(i / shardSize)].push(pkgs[i]);
|
||||
}
|
||||
return shards;
|
||||
}
|
||||
|
||||
// Parse command-line arguments, run main functionality, and exit.
|
||||
async function main() {
|
||||
const packages = await getPackages();
|
||||
const hash = await hashDockerfile(
|
||||
"composite",
|
||||
{
|
||||
"riju:runtime": await getLocalImageLabel(
|
||||
"riju:runtime",
|
||||
"riju.image-hash"
|
||||
),
|
||||
},
|
||||
{
|
||||
salt: {
|
||||
packageHashes: (
|
||||
await Promise.all(
|
||||
packages.map(async ({ debPath }) => {
|
||||
return (
|
||||
await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
|
||||
getStdout: true,
|
||||
})
|
||||
).stdout.trim();
|
||||
})
|
||||
)
|
||||
).sort(),
|
||||
},
|
||||
}
|
||||
);
|
||||
const server = getServer({
|
||||
shards: getShards(packages),
|
||||
});
|
||||
await new Promise((resolve) => server.listen(8487, "localhost", resolve));
|
||||
try {
|
||||
await runCommand(
|
||||
`docker build . -f docker/composite/Dockerfile -t riju:composite` +
|
||||
` --network host --no-cache --label riju.image-hash=${hash}`
|
||||
);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
import { promises as fs } from "fs";
|
||||
import http from "http";
|
||||
|
||||
import { Command } from "commander";
|
||||
import express from "express";
|
||||
|
||||
import { readLangConfig } from "./config.js";
|
||||
import { getLocalImageLabel } from "./docker-util.js";
|
||||
import { hashDockerfile } from "./hash-dockerfile.js";
|
||||
import { getDebHash, runCommand } from "./util.js";
|
||||
|
||||
// Get a Node.js http server object that will allow the Docker
|
||||
// build to fetch files from outside the container, without them
|
||||
// being in the build context.
|
||||
function getServer() {
|
||||
const app = express();
|
||||
app.use("/fs", express.static("."));
|
||||
return http.createServer(app);
|
||||
}
|
||||
|
||||
// Parse command-line arguments, run main functionality, and exit.
|
||||
async function main() {
|
||||
const program = new Command();
|
||||
program.requiredOption("--lang <id>", "language ID");
|
||||
program.option("--debug", "interactive debugging");
|
||||
program.parse(process.argv);
|
||||
const { lang, debug } = program;
|
||||
const hash = await hashDockerfile(
|
||||
"lang",
|
||||
{
|
||||
"riju:runtime": await getLocalImageLabel(
|
||||
"riju:runtime",
|
||||
"riju.image-hash"
|
||||
),
|
||||
},
|
||||
{
|
||||
salt: {
|
||||
langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`),
|
||||
sharedHashes: (
|
||||
await Promise.all(
|
||||
(((await readLangConfig(lang)).install || {}).riju || []).map(
|
||||
async (name) =>
|
||||
await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`)
|
||||
)
|
||||
)
|
||||
).sort(),
|
||||
},
|
||||
}
|
||||
);
|
||||
const server = getServer();
|
||||
await new Promise((resolve) => server.listen(8487, "localhost", resolve));
|
||||
try {
|
||||
if (debug) {
|
||||
await runCommand(
|
||||
`docker run -it --rm -e LANG=${lang} -w /tmp/riju-work --network host riju:runtime`
|
||||
);
|
||||
} else {
|
||||
await runCommand(
|
||||
`docker build . -f docker/lang/Dockerfile ` +
|
||||
`--build-arg LANG=${lang} -t riju:lang-${lang} ` +
|
||||
`--network host --no-cache --label riju.image-hash=${hash}`
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -187,8 +187,8 @@ async function main() {
|
|||
}
|
||||
const [name] = program.args;
|
||||
const { debug } = program.opts();
|
||||
if (name === "composite") {
|
||||
throw new Error("use build-composite-image.js instead for this");
|
||||
if (name === "lang") {
|
||||
throw new Error("use build-lang-image.js instead for this");
|
||||
}
|
||||
if (debug) {
|
||||
console.log(
|
||||
|
|
|
|||
|
|
@ -43,3 +43,12 @@ export async function runCommand(cmd, options) {
|
|||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
// Return the Riju-Script-Hash field in a .deb file generated by Riju.
|
||||
export async function getDebHash(debPath) {
|
||||
return (
|
||||
await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
|
||||
getStdout: true,
|
||||
})
|
||||
).stdout.trim();
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue