MirrorMirror
/
riju
mirror of https://github.com/radian-software/riju.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove top-level references to Depgraph hashing

This commit is contained in:
Radon Rosborough 2022-10-10 09:50:10 -07:00
parent c80a7acb89
commit 5fa070185a
3 changed files with 10 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Makefile
View File

@ -11,7 +11,6 @@ DEB := riju-$(T)-$(L).deb
S3 := s3://$(S3_BUCKET) S3 := s3://$(S3_BUCKET)
S3_CONFIG_PATH ?= config.json S3_CONFIG_PATH ?= config.json
S3_DEB := $(S3)/debs/$(DEB) S3_DEB := $(S3)/debs/$(DEB)
S3_HASH := $(S3)/hashes/riju-$(T)-$(L)
S3_CONFIG := $(S3)/$(S3_CONFIG_PATH) S3_CONFIG := $(S3)/$(S3_CONFIG_PATH)
ifneq ($(CMD),) ifneq ($(CMD),)
@ -52,13 +51,10 @@ image: # I=<image> [L=<lang>] [NC=1] : Build a Docker image
ifeq ($(I),lang) ifeq ($(I),lang)
@: $${L} @: $${L}
node tools/build-lang-image.js --lang $(L) node tools/build-lang-image.js --lang $(L)
else ifeq ($(I),ubuntu)
docker pull ubuntu:21.04
hash="$$(docker inspect ubuntu:21.04 -f '{{ .Id }}' | sha1sum | awk '{ print $$1 }')"; echo "FROM ubuntu:21.04" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) -
else ifneq (,$(filter $(I),admin ci)) else ifneq (,$(filter $(I),admin ci))
docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
else else
hash="$$(node tools/hash-dockerfile.js $(I) | grep .)"; docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --label riju.image-hash="$${hash}" $(NO_CACHE) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
endif endif
VOLUME_MOUNT ?= $(PWD) VOLUME_MOUNT ?= $(PWD)
@ -86,25 +82,22 @@ else
LANG_TAG := $(I) LANG_TAG := $(I)
endif endif
IMAGE_HASH := "$$(docker inspect riju:$(LANG_TAG) -f '{{ index .Config.Labels "riju.image-hash" }}')"
WITH_IMAGE_HASH := -e RIJU_IMAGE_HASH=$(IMAGE_HASH)
shell: # I=<shell> [L=<lang>] [E[E]=1] [P1|P2=<port>] [CMD="<arg>..."] : Launch Docker image with shell shell: # I=<shell> [L=<lang>] [E[E]=1] [P1|P2=<port>] [CMD="<arg>..."] : Launch Docker image with shell
@: $${I} @: $${I}
ifneq (,$(filter $(I),admin ci)) ifneq (,$(filter $(I),admin ci))
@mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d @mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d
docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/cache/riju/.aws -v $(HOME)/.docker:/var/cache/riju/.docker -v $(HOME)/.ssh:/var/cache/riju/.ssh -v $(HOME)/.terraform.d:/var/cache/riju/.terraform.d -e NI -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_REPO -e PUBLIC_DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/cache/riju/.aws -v $(HOME)/.docker:/var/cache/riju/.docker -v $(HOME)/.ssh:/var/cache/riju/.ssh -v $(HOME)/.terraform.d:/var/cache/riju/.terraform.d -e NI -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_REPO -e PUBLIC_DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD)
else ifeq ($(I),app) else ifeq ($(I),app)
docker run $(IT_ARG) --rm --hostname $(I) -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) docker run $(IT_ARG) --rm --hostname $(I) -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
else ifneq (,$(filter $(I),base lang)) else ifneq (,$(filter $(I),base lang))
ifeq ($(I),lang) ifeq ($(I),lang)
@: $${L} @: $${L}
endif endif
docker run $(IT_ARG) --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) docker run $(IT_ARG) --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD)
else ifeq ($(I),runtime) else ifeq ($(I),runtime)
docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
else else
docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
endif endif
ecr: # Authenticate to ECR (temporary credentials) ecr: # Authenticate to ECR (temporary credentials)
@ -242,21 +235,13 @@ undeploy: # Pull latest deployment config from S3
push: # I=<image> : Push Riju image to Docker registry push: # I=<image> : Push Riju image to Docker registry
@: $${I} $${DOCKER_REPO} @: $${I} $${DOCKER_REPO}
docker tag riju:$(I) $(DOCKER_REPO):$(I)-$(IMAGE_HASH)
docker push $(DOCKER_REPO):$(I)-$(IMAGE_HASH)
ifeq ($(I),ubuntu)
docker tag riju:$(I) $(PUBLIC_DOCKER_REPO):$(I)
docker push $(PUBLIC_DOCKER_REPO):$(I)
endif
docker tag riju:$(I) $(DOCKER_REPO):$(I) docker tag riju:$(I) $(DOCKER_REPO):$(I)
docker push $(DOCKER_REPO):$(I) docker push $(DOCKER_REPO):$(I)
upload: # L=<lang> T=<type> : Upload .deb to S3 upload: # L=<lang> T=<type> : Upload .deb to S3
@: $${L} $${T} $${S3_BUCKET} @: $${L} $${T} $${S3_BUCKET}
tools/ensure-deb-compressed.bash tools/ensure-deb-compressed.bash
aws s3 rm --recursive $(S3_HASH)
aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) aws s3 cp $(BUILD)/$(DEB) $(S3_DEB)
hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null
deploy-config: # Generate deployment config file deploy-config: # Generate deployment config file
node tools/generate-deploy-config.js node tools/generate-deploy-config.js

49
tools/build-lang-image.js
View File

@ -1,15 +1,10 @@
import crypto from "crypto";
import { promises as fs } from "fs";
import http from "http"; import http from "http";
import url from "url"; import url from "url";
import { Command } from "commander"; import { Command } from "commander";
import express from "express"; import express from "express";
import { getSharedDepsForLangConfig, readLangConfig } from "../lib/yaml.js"; import { runCommand } from "./util.js";
import { getLocalImageLabel } from "./docker-util.js";
import { hashDockerfile } from "./hash-dockerfile.js";
import { getDebHash, runCommand } from "./util.js";
// Get a Node.js http server object that will allow the Docker // Get a Node.js http server object that will allow the Docker
// build to fetch files from outside the container, without them // build to fetch files from outside the container, without them
@ -27,44 +22,6 @@ async function main() {
program.option("--debug", "interactive debugging"); program.option("--debug", "interactive debugging");
program.parse(process.argv); program.parse(process.argv);
const { lang, debug } = program.opts(); const { lang, debug } = program.opts();
const sharedDeps = await getSharedDepsForLangConfig(
await readLangConfig(lang)
);
const installContents = await fs.readFile(
`build/lang/${lang}/install.bash`,
"utf-8"
);
const sharedInstallContents = await Promise.all(
sharedDeps.map(async (name) =>
fs.readFile(`build/shared/${name}/install.bash`)
)
);
const allInstallContents = [].concat.apply(
[installContents],
sharedInstallContents
);
const hash = await hashDockerfile(
"lang",
{
"riju:base": await getLocalImageLabel("riju:base", "riju.image-hash"),
},
{
salt: {
langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`),
sharedHashes: (
await Promise.all(
sharedDeps.map(
async (name) =>
await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`)
)
)
).sort(),
installHash: allInstallContents
.map((c) => crypto.createHash("sha1").update(c).digest("hex"))
.join(""),
},
}
);
const server = getServer(); const server = getServer();
await new Promise((resolve) => server.listen(8487, "localhost", resolve)); await new Promise((resolve) => server.listen(8487, "localhost", resolve));
try { try {
@ -76,11 +33,11 @@ async function main() {
await runCommand( await runCommand(
`docker build . -f docker/lang/Dockerfile ` + `docker build . -f docker/lang/Dockerfile ` +
`--build-arg LANG=${lang} -t riju:lang-${lang} ` + `--build-arg LANG=${lang} -t riju:lang-${lang} ` +
`--network host --no-cache --label riju.image-hash=${hash}` `--network host --no-cache`
); );
} }
} finally { } finally {
await server.close(); await new Promise((resolve) => server.close(resolve));
} }
process.exit(0); process.exit(0);
} }

3
tools/generate-build-script.js
View File

@ -316,8 +316,7 @@ Maintainer: Radian LLC <contact+riju@radian.codes>
Description: The ${name} ${ Description: The ${name} ${
isShared ? "shared dependency" : "language" isShared ? "shared dependency" : "language"
} packaged for Riju } packaged for Riju
Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g') Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g')`;
Riju-Script-Hash: \$((cat "\$0"; echo "\${RIJU_IMAGE_HASH}") | sha1sum - | awk '{ print \$1 }')`;
parts.push(`\ parts.push(`\
install -d "\${pkg}/DEBIAN" install -d "\${pkg}/DEBIAN"
cat <<EOF > "\${pkg}/DEBIAN/control" cat <<EOF > "\${pkg}/DEBIAN/control"