diff --git a/docker/composite/Dockerfile b/docker/composite/Dockerfile index 1daece3..3e909d2 100644 --- a/docker/composite/Dockerfile +++ b/docker/composite/Dockerfile @@ -1,4 +1,16 @@ FROM riju:runtime COPY docker/composite/install.bash /tmp/ -RUN /tmp/install.bash + +# The number of commands here must match NUM_SHARDS in +# build-composite-image.js. +RUN /tmp/install.bash 0 +RUN /tmp/install.bash 1 +RUN /tmp/install.bash 2 +RUN /tmp/install.bash 3 +RUN /tmp/install.bash 4 +RUN /tmp/install.bash 5 +RUN /tmp/install.bash 6 +RUN /tmp/install.bash 7 +RUN /tmp/install.bash 8 +RUN /tmp/install.bash 9 diff --git a/docker/composite/install.bash b/docker/composite/install.bash index ef329bc..fae80fe 100755 --- a/docker/composite/install.bash +++ b/docker/composite/install.bash @@ -2,6 +2,8 @@ set -euxo pipefail +shard="$1" + function riju-curl { curl -fsSL "localhost:8487$1" } @@ -17,13 +19,8 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -riju-curl /shared | while read lang; do - riju-apt-install "/fs/build/shared/${lang}/riju-shared-${lang}.deb" -done - -riju-curl /langs | while read lang; do - riju-apt-install "/fs/build/lang/${lang}/riju-lang-${lang}.deb" - riju-apt-install "/fs/build/config/${lang}/riju-config-${lang}.deb" +riju-curl "/shard/${shard}" | while read path; do + riju-apt-install "/fs/${path}" done rm -rf *.deb diff --git a/tools/build-composite-image.js b/tools/build-composite-image.js index 0d152a4..218629e 100644 --- a/tools/build-composite-image.js +++ b/tools/build-composite-image.js @@ -8,23 +8,50 @@ import { getLocalImageLabel } from "./docker-util.js"; import { hashDockerfile } from "./hash-dockerfile.js"; import { runCommand } from "./util.js"; +// Number of package installation layers in the composite Docker +// image. This needs to match the number of installation RUN commands +// in the composite Dockerfile. +const NUM_SHARDS = 10; + // Get a Node.js http server object that will serve information and // files for packages that should be installed into the composite // Docker image. -function getServer({ langs, sharedDeps }) { +function getServer({ shards }) { const app = express(); - app.get("/langs", (req, res) => { - res.send(langs.map((lang) => lang + "\n").join("")); - }); - app.get("/shared", (req, res) => { - res.send(sharedDeps.map((lang) => lang + "\n").join("")); + app.get("/shard/:shard", (req, res) => { + res.send( + shards[parseInt(req.params.shard)] + .map(({ debPath }) => debPath + "\n") + .join("") + ); }); app.use("/fs", express.static(".")); return http.createServer(app); } +// Given a list of the packages to be built, split them into shards. +// Return a list of shards. Each shard is a list of the package +// objects, such that there are NUM_SHARDS shards. Traversing each +// shard in order will return the packages in the same order as the +// original list. +// +// Currently this uses an extremely simple algorithm, but that might +// be improved in the future. +function getShards(pkgs) { + const shards = []; + for (let i = 0; i < NUM_SHARDS; ++i) { + shards.push([]); + } + const shardSize = Math.ceil(pkgs.length / NUM_SHARDS); + for (let i = 0; i < pkgs.length; ++i) { + shards[Math.floor(i / shardSize)].push(pkgs[i]); + } + return shards; +} + // Parse command-line arguments, run main functionality, and exit. async function main() { + const packages = await getPackages(); const hash = await hashDockerfile( "composite", { @@ -37,7 +64,7 @@ async function main() { salt: { packageHashes: ( await Promise.all( - (await getPackages()).map(async ({ debPath }) => { + packages.map(async ({ debPath }) => { return ( await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, { getStdout: true, @@ -50,8 +77,7 @@ async function main() { } ); const server = getServer({ - langs: await getLangs(), - sharedDeps: await getSharedDeps(), + shards: getShards(packages), }); await new Promise((resolve) => server.listen(8487, "localhost", resolve)); try {