Start work on depgraph runner
This commit is contained in:
parent
b99d17bcd3
commit
219fbed342
38
Makefile
38
Makefile
|
@ -41,6 +41,9 @@ image: # I=<image> [L=<lang>] [NC=1] : Build a Docker image
|
||||||
ifeq ($(I),lang)
|
ifeq ($(I),lang)
|
||||||
@: $${L}
|
@: $${L}
|
||||||
node tools/build-lang-image.js --lang $(L)
|
node tools/build-lang-image.js --lang $(L)
|
||||||
|
else ifeq ($(I),ubuntu)
|
||||||
|
docker pull ubuntu:rolling
|
||||||
|
docker tag ubuntu:rolling riju:ubuntu
|
||||||
else ifneq (,$(filter $(I),admin ci))
|
else ifneq (,$(filter $(I),admin ci))
|
||||||
docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
|
docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE)
|
||||||
else
|
else
|
||||||
|
@ -66,21 +69,23 @@ else
|
||||||
LANG_TAG := $(I)
|
LANG_TAG := $(I)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
IMAGE_HASH := -e RIJU_IMAGE_HASH="$$(docker inspect riju:$(LANG_TAG) | jq '.[0].Config.Labels["riju.image-hash"]' -r)"
|
||||||
|
|
||||||
shell: # I=<shell> [L=<lang>] [E=1] [P1|P2=<port>] : Launch Docker image with shell
|
shell: # I=<shell> [L=<lang>] [E=1] [P1|P2=<port>] : Launch Docker image with shell
|
||||||
@: $${I}
|
@: $${I}
|
||||||
ifneq (,$(filter $(I),admin ci))
|
ifneq (,$(filter $(I),admin ci))
|
||||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD)
|
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/riju/.aws -v $(HOME)/.docker:/var/riju/.docker -v $(HOME)/.ssh:/var/riju/.ssh -v $(HOME)/.terraform.d:/var/riju/.terraform.d -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_USERNAME -e DOCKER_PASSWORD -e DEPLOY_SSH_PRIVATE_KEY -e DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) --network host riju:$(I) $(BASH_CMD)
|
||||||
else ifeq ($(I),app)
|
else ifeq ($(I),app)
|
||||||
docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
|
docker run -it --rm --hostname $(I) $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD)
|
||||||
else ifneq (,$(filter $(I),base lang))
|
else ifneq (,$(filter $(I),base lang))
|
||||||
ifeq ($(I),lang)
|
ifeq ($(I),lang)
|
||||||
@: $${L}
|
@: $${L}
|
||||||
endif
|
endif
|
||||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD)
|
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src --label riju-install-target=yes $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD)
|
||||||
else ifeq ($(I),runtime)
|
else ifeq ($(I),runtime)
|
||||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
|
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD)
|
||||||
else
|
else
|
||||||
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD)
|
docker run -it --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(IMAGE_HASH) riju:$(I) $(BASH_CMD)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
## This is equivalent to 'make pkg' in a fresh packaging container
|
## This is equivalent to 'make pkg' in a fresh packaging container
|
||||||
|
@ -194,11 +199,12 @@ dev: # Compile, run, and watch all artifacts and server for development
|
||||||
|
|
||||||
### Application tools
|
### Application tools
|
||||||
|
|
||||||
## L can be a language identifier or a test type (run, repl, lsp,
|
## L is a language identifier or a comma-separated list of them, to
|
||||||
## format, etc.). Multiple identifiers can be separated by spaces to
|
## filter tests by language. T is a test type (run, repl, lsp, format,
|
||||||
## form a conjunction (AND), or by commas to form a disjunction (OR).
|
## etc.) or a set of them to filter tests that way. If both filters
|
||||||
|
## are provided, then only tests matching both are run.
|
||||||
|
|
||||||
test: # L=<filter> : Run test(s) for language or test category
|
test: # [L=<lang>[,...]] [T=<test>[,...]] : Run test(s) for language or test category
|
||||||
node backend/test-runner.js $(L)
|
node backend/test-runner.js $(L)
|
||||||
|
|
||||||
## Functions such as 'repl', 'run', 'format', etc. are available in
|
## Functions such as 'repl', 'run', 'format', etc. are available in
|
||||||
|
@ -219,9 +225,6 @@ lsp: # L=<lang|cmd> : Run LSP REPL for language or custom command line
|
||||||
|
|
||||||
### Fetch artifacts from registries
|
### Fetch artifacts from registries
|
||||||
|
|
||||||
pull-base: # Pull latest base image(s) from Docker Hub
|
|
||||||
docker pull ubuntu:rolling
|
|
||||||
|
|
||||||
pull: # I=<image> : Pull last published Riju image from Docker Hub
|
pull: # I=<image> : Pull last published Riju image from Docker Hub
|
||||||
@: $${I} $${DOCKER_REPO}
|
@: $${I} $${DOCKER_REPO}
|
||||||
docker pull $(DOCKER_REPO):$(I)
|
docker pull $(DOCKER_REPO):$(I)
|
||||||
|
@ -232,12 +235,6 @@ download: # L=<lang> T=<type> : Download last published .deb from S3
|
||||||
mkdir -p $(BUILD)
|
mkdir -p $(BUILD)
|
||||||
aws s3 cp $(S3_DEB) $(BUILD)/$(DEB) --no-sign-request
|
aws s3 cp $(S3_DEB) $(BUILD)/$(DEB) --no-sign-request
|
||||||
|
|
||||||
plan: # Display plan to pull/rebuild outdated or missing artifacts
|
|
||||||
node tools/plan-publish.js
|
|
||||||
|
|
||||||
sync: # Pull/rebuild outdated or missing artifacts
|
|
||||||
node tools/plan-publish.js --execute
|
|
||||||
|
|
||||||
### Publish artifacts to registries
|
### Publish artifacts to registries
|
||||||
|
|
||||||
push: # I=<image> : Push Riju image to Docker Hub
|
push: # I=<image> : Push Riju image to Docker Hub
|
||||||
|
@ -251,11 +248,6 @@ upload: # L=<lang> T=<type> : Upload .deb to S3
|
||||||
aws s3 cp $(BUILD)/$(DEB) $(S3_DEB)
|
aws s3 cp $(BUILD)/$(DEB) $(S3_DEB)
|
||||||
hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null
|
hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null
|
||||||
|
|
||||||
## You should probably only run this from CI.
|
|
||||||
|
|
||||||
publish: # Full synchronization and prod deployment
|
|
||||||
tools/publish.bash
|
|
||||||
|
|
||||||
### Miscellaneous
|
### Miscellaneous
|
||||||
|
|
||||||
## Run this every time you update .gitignore or .dockerignore.in.
|
## Run this every time you update .gitignore or .dockerignore.in.
|
||||||
|
|
|
@ -6,6 +6,7 @@ import pQueue from "p-queue";
|
||||||
const PQueue = pQueue.default;
|
const PQueue = pQueue.default;
|
||||||
import stripAnsi from "strip-ansi";
|
import stripAnsi from "strip-ansi";
|
||||||
|
|
||||||
|
import { getTestHash } from "../lib/hash-test.js";
|
||||||
import * as api from "./api.js";
|
import * as api from "./api.js";
|
||||||
import { langsPromise } from "./langs.js";
|
import { langsPromise } from "./langs.js";
|
||||||
import { getUUID } from "./util.js";
|
import { getUUID } from "./util.js";
|
||||||
|
@ -623,15 +624,11 @@ async function main() {
|
||||||
langs = await langsPromise;
|
langs = await langsPromise;
|
||||||
let tests = getTestList();
|
let tests = getTestList();
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
for (const arg of args) {
|
if (process.env.L) {
|
||||||
tests = tests.filter(
|
tests = tests.filter(({ lang }) => process.env.L.split().includes(lang));
|
||||||
({ lang, type }) =>
|
}
|
||||||
arg
|
if (process.env.T) {
|
||||||
.split(",")
|
tests = tests.filter(({ type }) => process.env.T.split().includes(type));
|
||||||
.filter((arg) =>
|
|
||||||
[lang, type].concat(langs[lang].aliases || []).includes(arg)
|
|
||||||
).length > 0
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if (tests.length === 0) {
|
if (tests.length === 0) {
|
||||||
console.error("no tests selected");
|
console.error("no tests selected");
|
||||||
|
@ -732,6 +729,23 @@ async function main() {
|
||||||
console.error(` - ${lang}/${type} (${err})`)
|
console.error(` - ${lang}/${type} (${err})`)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
const langsValidated = {};
|
||||||
|
passed.forEach((_, { lang }) => {
|
||||||
|
langsValidated[lang] = true;
|
||||||
|
});
|
||||||
|
failed.forEach(({ lang }) => {
|
||||||
|
langsValidated[lang] = false;
|
||||||
|
});
|
||||||
|
for (const [lang, validated] of Object.entries(langsValidated)) {
|
||||||
|
if (!validated) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
await fs.mkdir(`build/test-hashes/lang`, { recursive: true });
|
||||||
|
await fs.writeFile(
|
||||||
|
`build/test-hashes/lang/${lang}`,
|
||||||
|
await getTestHash(lang)
|
||||||
|
);
|
||||||
|
}
|
||||||
process.exit(failed.size > 0 ? 1 : 0);
|
process.exit(failed.size > 0 ? 1 : 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
exec node "$(dirname "$(realpath "$0")")"/../tools/depgraph.js "$@"
|
|
@ -0,0 +1,83 @@
|
||||||
|
import crypto from "crypto";
|
||||||
|
import { promises as fs } from "fs";
|
||||||
|
import path from "path";
|
||||||
|
|
||||||
|
import { parse } from "@babel/parser";
|
||||||
|
import { simple as babelWalk } from "babel-walk";
|
||||||
|
|
||||||
|
import { readLangConfig } from "./yaml.js";
|
||||||
|
|
||||||
|
async function getRelativeImports(filename) {
|
||||||
|
const relativeImports = [];
|
||||||
|
const program = parse(await fs.readFile(filename, "utf-8"), {
|
||||||
|
sourceType: "module",
|
||||||
|
plugins: ["classProperties"],
|
||||||
|
});
|
||||||
|
babelWalk({
|
||||||
|
ImportDeclaration: (node) => {
|
||||||
|
if (node.source.type !== "StringLiteral") {
|
||||||
|
throw new Error(`unsupported import syntax:`, node);
|
||||||
|
}
|
||||||
|
const source = node.source.value;
|
||||||
|
if (!source.startsWith(".")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
relativeImports.push(source);
|
||||||
|
},
|
||||||
|
})(program);
|
||||||
|
return relativeImports;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pathRelativeTo(relativePath, relativeTo) {
|
||||||
|
return path.join(path.dirname(path.resolve(relativeTo)), relativePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getTransitiveRelativeImports(filename) {
|
||||||
|
let queue = [filename];
|
||||||
|
const found = new Set();
|
||||||
|
while (queue.length > 0) {
|
||||||
|
const filename = path.resolve(queue.pop());
|
||||||
|
if (found.has(filename)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
found.add(filename);
|
||||||
|
queue = queue.concat(
|
||||||
|
(await getRelativeImports(filename)).map((result) =>
|
||||||
|
pathRelativeTo(result, filename)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return [...found];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getTestRunnerHash() {
|
||||||
|
const files = await getTransitiveRelativeImports("backend/test-runner.js");
|
||||||
|
files.push("package.json");
|
||||||
|
files.push("yarn.lock");
|
||||||
|
const hashes = [];
|
||||||
|
for (const file of files) {
|
||||||
|
hashes.push(
|
||||||
|
crypto
|
||||||
|
.createHash("sha1")
|
||||||
|
.update(await fs.readFile(file, "utf-8"))
|
||||||
|
.digest("hex")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return crypto.createHash("sha1").update(hashes.join(",")).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
const testRunnerHash = getTestRunnerHash();
|
||||||
|
|
||||||
|
async function getTestConfigHash(lang) {
|
||||||
|
const config = Object.assign({}, await readLangConfig(lang));
|
||||||
|
delete config["install"];
|
||||||
|
delete config["info"];
|
||||||
|
return crypto.createHash("sha1").update(JSON.stringify(config)).digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getTestHash(lang) {
|
||||||
|
return crypto
|
||||||
|
.createHash("sha1")
|
||||||
|
.update(`${await testRunnerHash},${await getTestConfigHash(lang)}`)
|
||||||
|
.digest("hex");
|
||||||
|
}
|
|
@ -114,3 +114,10 @@ export async function readSharedDepConfig(lang) {
|
||||||
}
|
}
|
||||||
return fixupLangConfig(langConfig);
|
return fixupLangConfig(langConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Given a language config JSON, return a list of the Riju shared
|
||||||
|
// dependency names, or an empty list if none are configured for this
|
||||||
|
// language.
|
||||||
|
export async function getSharedDepsForLangConfig(langConfig) {
|
||||||
|
return (langConfig.install && langConfig.install.riju) || [];
|
||||||
|
}
|
||||||
|
|
|
@ -6,12 +6,14 @@
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@babel/core": "^7.12.10",
|
"@babel/core": "^7.12.10",
|
||||||
|
"@babel/parser": "^7.13.11",
|
||||||
"@babel/preset-env": "^7.12.11",
|
"@babel/preset-env": "^7.12.11",
|
||||||
"@balena/dockerignore": "^1.0.2",
|
"@balena/dockerignore": "^1.0.2",
|
||||||
"async-lock": "^1.2.6",
|
"async-lock": "^1.2.6",
|
||||||
"babel-loader": "^8.2.2",
|
"babel-loader": "^8.2.2",
|
||||||
|
"babel-walk": "^3.0.0",
|
||||||
"buffer": "^6.0.3",
|
"buffer": "^6.0.3",
|
||||||
"commander": "^6.2.1",
|
"commander": "^7.1.0",
|
||||||
"css-loader": "^5.0.1",
|
"css-loader": "^5.0.1",
|
||||||
"debounce": "^1.2.0",
|
"debounce": "^1.2.0",
|
||||||
"docker-file-parser": "^1.0.5",
|
"docker-file-parser": "^1.0.5",
|
||||||
|
|
|
@ -4,7 +4,7 @@ import http from "http";
|
||||||
import { Command } from "commander";
|
import { Command } from "commander";
|
||||||
import express from "express";
|
import express from "express";
|
||||||
|
|
||||||
import { readLangConfig } from "../lib/yaml.js";
|
import { getSharedDepsForLangConfig, readLangConfig } from "../lib/yaml.js";
|
||||||
import { getLocalImageLabel } from "./docker-util.js";
|
import { getLocalImageLabel } from "./docker-util.js";
|
||||||
import { hashDockerfile } from "./hash-dockerfile.js";
|
import { hashDockerfile } from "./hash-dockerfile.js";
|
||||||
import { getDebHash, runCommand } from "./util.js";
|
import { getDebHash, runCommand } from "./util.js";
|
||||||
|
@ -35,7 +35,7 @@ async function main() {
|
||||||
langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`),
|
langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`),
|
||||||
sharedHashes: (
|
sharedHashes: (
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
(((await readLangConfig(lang)).install || {}).riju || []).map(
|
getSharedDepsForLangConfig(await readLangConfig(lang)).map(
|
||||||
async (name) =>
|
async (name) =>
|
||||||
await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`)
|
await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`)
|
||||||
)
|
)
|
||||||
|
|
|
@ -0,0 +1,304 @@
|
||||||
|
import crypto from "crypto";
|
||||||
|
import { promises as fs } from "fs";
|
||||||
|
import process from "process";
|
||||||
|
import url from "url";
|
||||||
|
|
||||||
|
import { Command } from "commander";
|
||||||
|
|
||||||
|
import { getTestHash } from "../lib/hash-test.js";
|
||||||
|
import {
|
||||||
|
getLangs,
|
||||||
|
getSharedDeps,
|
||||||
|
getSharedDepsForLangConfig,
|
||||||
|
readLangConfig,
|
||||||
|
} from "../lib/yaml.js";
|
||||||
|
import {
|
||||||
|
getDockerRepo,
|
||||||
|
getLocalImageLabel,
|
||||||
|
getRemoteImageLabel,
|
||||||
|
} from "./docker-util.js";
|
||||||
|
import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js";
|
||||||
|
import { runCommand } from "./util.js";
|
||||||
|
|
||||||
|
function getS3Bucket() {
|
||||||
|
if (!process.env.S3_BUCKET) {
|
||||||
|
throw new Error(`unset environment variable: \$S3_BUCKET`);
|
||||||
|
}
|
||||||
|
return process.env.S3_BUCKET;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getInformationalDependencies() {
|
||||||
|
return {
|
||||||
|
s3DebHashes: async () => {
|
||||||
|
return Object.fromEntries(
|
||||||
|
JSON.parse(
|
||||||
|
(
|
||||||
|
await runCommand(
|
||||||
|
`aws s3api list-objects-v2 --bucket riju-debs --prefix hashes`,
|
||||||
|
{ getStdout: true }
|
||||||
|
)
|
||||||
|
).stdout
|
||||||
|
).Contents.map(({ Key: key }) => {
|
||||||
|
const [_, remoteName, remoteHash] = key.split("/");
|
||||||
|
return [remoteName, remoteHash];
|
||||||
|
})
|
||||||
|
);
|
||||||
|
},
|
||||||
|
s3TestHashes: async () => {
|
||||||
|
return Object.fromEntries(
|
||||||
|
JSON.parse(
|
||||||
|
(
|
||||||
|
await runCommand(
|
||||||
|
`aws s3api list-objects-v2 --bucket riju-debs --prefix test-hashes/lang`,
|
||||||
|
{ getStdout: true }
|
||||||
|
)
|
||||||
|
).stdout
|
||||||
|
).Contents.map(({ Key: key }) => {
|
||||||
|
const [_1, _2, remoteName, remoteHash] = key.split("/");
|
||||||
|
return [remoteName, remoteHash];
|
||||||
|
})
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getImageArtifact({ tag, isBaseImage, isLangImage }) {
|
||||||
|
const DOCKER_REPO = getDockerRepo();
|
||||||
|
const name = isLangImage ? "lang" : tag;
|
||||||
|
let baseImageTags = [];
|
||||||
|
let dependencies = [];
|
||||||
|
if (!isBaseImage) {
|
||||||
|
baseImageTags = [...new Set(await getBaseImages(name))].map((baseImage) => {
|
||||||
|
if (!baseImage.startsWith("riju:")) {
|
||||||
|
throw new Error(
|
||||||
|
`non-Riju base image '${baseImage}' in Dockerfile for ${name} image`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return baseImage.replace(/^riju:/, "");
|
||||||
|
});
|
||||||
|
dependencies = baseImageTags.map(
|
||||||
|
(baseImageName) => `image:${baseImageTag}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (isLangImage) {
|
||||||
|
dependencies.push(`deb:lang-${isLangImage.lang}`);
|
||||||
|
dependencies.concat(
|
||||||
|
isLangImage.sharedDeps.map((name) => `deb:shared-${name}`)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
name: `image:${tag}`,
|
||||||
|
dependencies: dependencies,
|
||||||
|
getLocalHash: async () => {
|
||||||
|
return await getLocalImageLabel(`riju:${tag}`, "riju.image-hash");
|
||||||
|
},
|
||||||
|
getPublishedHash: async () => {
|
||||||
|
return await getRemoteImageLabel(
|
||||||
|
`${DOCKER_REPO}:${tag}`,
|
||||||
|
"riju.image-hash"
|
||||||
|
);
|
||||||
|
},
|
||||||
|
getDesiredHash: async (dependencyHashes) => {
|
||||||
|
if (isBaseImage) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const dependentDockerHashes = {};
|
||||||
|
for (const baseImageTag of baseImageTag) {
|
||||||
|
dependentDockerHashes[`riju:${baseImageTag}`] =
|
||||||
|
dependencyHashes[`image:${baseImageTag}`];
|
||||||
|
}
|
||||||
|
const salt = null;
|
||||||
|
if (isLangImage) {
|
||||||
|
salt.langHash = dependencyHashes[`deb:lang-${isLangImage.lang}`];
|
||||||
|
salt.sharedHashes = isLangImage.sharedDeps.map(
|
||||||
|
(name) => dependencyHashes[`deb:shared-${name}`]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return await hashDockerfile(name, dependentDockerHashes, { salt });
|
||||||
|
},
|
||||||
|
buildLocally: async () => {
|
||||||
|
await runCommand(`make image I=${tag}`);
|
||||||
|
},
|
||||||
|
retrieveFromRegistry: async () => {
|
||||||
|
await runCommand(`make pull I=${tag}`);
|
||||||
|
},
|
||||||
|
publishToRegistry: async () => {
|
||||||
|
await runCommand(`make push I=${tag}`);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getDebArtifact({ type, lang }) {
|
||||||
|
return {
|
||||||
|
name: `deb:${type}-${lang}`,
|
||||||
|
dependencies: ["image:packaging"],
|
||||||
|
informationalDependencies: {
|
||||||
|
getPublishedHash: "s3DebHashes",
|
||||||
|
},
|
||||||
|
getLocalHash: async () => {
|
||||||
|
try {
|
||||||
|
await fs.access(debPath);
|
||||||
|
} catch (err) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
(
|
||||||
|
await runCommand(`dpkg-deb -f ${debPath} Riju-Script-Hash`, {
|
||||||
|
getStdout: true,
|
||||||
|
})
|
||||||
|
).stdout.trim() || null
|
||||||
|
);
|
||||||
|
},
|
||||||
|
getPublishedHash: async ({ s3DebHashes }) => {
|
||||||
|
return s3DebHashes[`riju-${type}-${lang}`] || null;
|
||||||
|
},
|
||||||
|
getDesiredHash: async () => {
|
||||||
|
let contents = await fs.readFile(
|
||||||
|
`build/${type}/${lang}/build.bash`,
|
||||||
|
"utf-8"
|
||||||
|
);
|
||||||
|
contents +=
|
||||||
|
(await getLocalImageLabel("riju:packaging", "riju.image-hash")) + "\n";
|
||||||
|
return crypto.createHash("sha1").update(contents).digest("hex");
|
||||||
|
},
|
||||||
|
buildLocally: async () => {
|
||||||
|
await runCommand(
|
||||||
|
`make shell I=packaging CMD="make pkg T=${type} L=${lang}"`
|
||||||
|
);
|
||||||
|
},
|
||||||
|
retrieveFromRegistry: async () => {
|
||||||
|
await runCommand(`make download T=${type} L=${lang}`);
|
||||||
|
},
|
||||||
|
publishToRegistry: async () => {
|
||||||
|
await runCommand(`make upload T=${type} L=${lang}`);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getLanguageTestArtifact({ lang }) {
|
||||||
|
return {
|
||||||
|
name: `test:lang-${lang}`,
|
||||||
|
dependencies: ["image:runtime"],
|
||||||
|
informationalDependencies: {
|
||||||
|
getPublishedHash: "s3TestHashes",
|
||||||
|
retrieveFromRegistry: "s3TestHashes",
|
||||||
|
},
|
||||||
|
getLocalHash: async () => {
|
||||||
|
const hashPath = `build/test-hashes/lang/${lang}`;
|
||||||
|
let hash;
|
||||||
|
try {
|
||||||
|
return (await fs.readFile(hashPath, "utf-8")).trim();
|
||||||
|
} catch (err) {
|
||||||
|
if (err.code === "ENOENT") {
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
getPublishedHash: async ({ s3TestHashes }) => {
|
||||||
|
return s3TestHashes[lang];
|
||||||
|
},
|
||||||
|
getDesiredHash: async () => {
|
||||||
|
return await getTestHash(lang);
|
||||||
|
},
|
||||||
|
buildLocally: async () => {
|
||||||
|
await runCommand(`make shell I=runtime CMD="make test L=${lang}"`);
|
||||||
|
},
|
||||||
|
retrieveFromRegistry: async ({ s3TestHashes }) => {
|
||||||
|
await fs.writeFile(
|
||||||
|
`build/test-hashes/lang/${lang}`,
|
||||||
|
s3TestHashes[lang] + "\n"
|
||||||
|
);
|
||||||
|
},
|
||||||
|
publishToRegistry: async () => {
|
||||||
|
const hashPath = `build/test-hashes/lang/${lang}`;
|
||||||
|
const hash = (await fs.readFile(hashPath, "utf-8")).trim();
|
||||||
|
const S3_BUCKET = getS3Bucket();
|
||||||
|
await runCommand(
|
||||||
|
`aws s3 cp ${hashPath} s3://${S3_BUCKET}/test-hashes/lang/${lang}/${hash}`
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getDeployArtifact(langs) {
|
||||||
|
return {
|
||||||
|
name: `deploy:prod`,
|
||||||
|
dependencies: ["image:app"]
|
||||||
|
.concat(langs.map((lang) => `image:lang-${lang}`))
|
||||||
|
.concat(langs.map((lang) => `test:lang-${lang}`)),
|
||||||
|
getLocalHash: async () => {
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getDepGraph() {
|
||||||
|
const informationalDependencies = getInformationalDependencies();
|
||||||
|
const artifacts = [];
|
||||||
|
artifacts.push(
|
||||||
|
await getImageArtifact({
|
||||||
|
tag: "ubuntu",
|
||||||
|
isBaseImage: true,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
artifacts.push(await getImageArtifact({ tag: "packaging" }));
|
||||||
|
artifacts.push(await getImageArtifact({ tag: "base" }));
|
||||||
|
for (const sharedDep of await getSharedDeps()) {
|
||||||
|
artifacts.push(await getDebArtifact({ type: "shared", lang: sharedDep }));
|
||||||
|
}
|
||||||
|
const langs = await getLangs();
|
||||||
|
const langConfigs = Object.fromEntries(
|
||||||
|
await Promise.all(
|
||||||
|
langs.map(async (lang) => [lang, await readLangConfig(lang)])
|
||||||
|
)
|
||||||
|
);
|
||||||
|
artifacts.push(await getImageArtifact({ tag: "runtime" }));
|
||||||
|
for (const lang of langs) {
|
||||||
|
artifacts.push(await getDebArtifact({ type: "lang", lang: lang }));
|
||||||
|
artifacts.push(
|
||||||
|
await getImageArtifact({
|
||||||
|
tag: `lang-${lang}`,
|
||||||
|
isLangImage: {
|
||||||
|
lang: lang,
|
||||||
|
sharedDeps: await getSharedDepsForLangConfig(langConfigs[lang]),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
artifacts.push(await getLanguageTestArtifact({ lang: lang }));
|
||||||
|
}
|
||||||
|
artifacts.push(await getImageArtifact({ tag: "app" }));
|
||||||
|
artifacts.push(await getDeployArtifact(langs));
|
||||||
|
return { informationalDependencies, artifacts };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const program = new Command();
|
||||||
|
program.usage("<target>...");
|
||||||
|
program.option("--list", "list available artifacts; ignore other arguments");
|
||||||
|
program.option("--publish", "publish artifacts to remote registries");
|
||||||
|
program.option("--yes", "execute plan without confirmation");
|
||||||
|
program.parse(process.argv);
|
||||||
|
const { list, publish, yes } = program.opts();
|
||||||
|
const depgraph = await getDepGraph();
|
||||||
|
if (list) {
|
||||||
|
for (const { name } of depgraph.artifacts) {
|
||||||
|
console.log(name);
|
||||||
|
}
|
||||||
|
console.error();
|
||||||
|
console.error(`${depgraph.artifacts.length} artifacts`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
if (program.args.length === 0) {
|
||||||
|
program.help({ error: true });
|
||||||
|
}
|
||||||
|
console.log("doing things now");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
|
@ -254,7 +254,7 @@ Description: The ${name} ${
|
||||||
isShared ? "shared dependency" : "language"
|
isShared ? "shared dependency" : "language"
|
||||||
} packaged for Riju
|
} packaged for Riju
|
||||||
Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g')
|
Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g')
|
||||||
Riju-Script-Hash: \$(sha1sum "\$0" | awk '{ print \$1 }')`;
|
Riju-Script-Hash: \$((cat "\$0"; echo "\${RIJU_IMAGE_HASH}") | sha1sum - | awk '{ print \$1 }')`;
|
||||||
parts.push(`\
|
parts.push(`\
|
||||||
install -d "\${pkg}/DEBIAN"
|
install -d "\${pkg}/DEBIAN"
|
||||||
cat <<EOF > "\${pkg}/DEBIAN/control"
|
cat <<EOF > "\${pkg}/DEBIAN/control"
|
||||||
|
|
|
@ -64,6 +64,24 @@ async function listFiles(path) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getBaseImages(name) {
|
||||||
|
const dockerfile = await parseDockerfile(name);
|
||||||
|
const baseImages = [];
|
||||||
|
dockerfile.map(({ name, args, error }) => {
|
||||||
|
if (error) {
|
||||||
|
throw error;
|
||||||
|
if (name === "FROM") {
|
||||||
|
if (typeof args !== "string") {
|
||||||
|
throw new Error("got unexpected non-string for FROM args");
|
||||||
|
}
|
||||||
|
const image = args.split(" ")[0];
|
||||||
|
baseImages.push(image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return baseImages;
|
||||||
|
}
|
||||||
|
|
||||||
// Given a Dockerfile name like "packaging", read all the necessary
|
// Given a Dockerfile name like "packaging", read all the necessary
|
||||||
// files from disk and then convert the Dockerfile into a JavaScript
|
// files from disk and then convert the Dockerfile into a JavaScript
|
||||||
// object which includes all relevant build context. The idea is that
|
// object which includes all relevant build context. The idea is that
|
||||||
|
|
28
yarn.lock
28
yarn.lock
|
@ -251,6 +251,11 @@
|
||||||
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.11.tgz#9ce3595bcd74bc5c466905e86c535b8b25011e79"
|
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.11.tgz#9ce3595bcd74bc5c466905e86c535b8b25011e79"
|
||||||
integrity sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg==
|
integrity sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg==
|
||||||
|
|
||||||
|
"@babel/parser@^7.13.11":
|
||||||
|
version "7.13.11"
|
||||||
|
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.11.tgz#f93ebfc99d21c1772afbbaa153f47e7ce2f50b88"
|
||||||
|
integrity sha512-PhuoqeHoO9fc4ffMEVk4qb/w/s2iOSWohvbHxLtxui0eBg3Lg5gN1U8wp1V1u61hOWkPQJJyJzGH6Y+grwkq8Q==
|
||||||
|
|
||||||
"@babel/plugin-proposal-async-generator-functions@^7.12.1":
|
"@babel/plugin-proposal-async-generator-functions@^7.12.1":
|
||||||
version "7.12.12"
|
version "7.12.12"
|
||||||
resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.12.tgz#04b8f24fd4532008ab4e79f788468fd5a8476566"
|
resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.12.tgz#04b8f24fd4532008ab4e79f788468fd5a8476566"
|
||||||
|
@ -814,6 +819,15 @@
|
||||||
lodash "^4.17.19"
|
lodash "^4.17.19"
|
||||||
to-fast-properties "^2.0.0"
|
to-fast-properties "^2.0.0"
|
||||||
|
|
||||||
|
"@babel/types@^7.9.6":
|
||||||
|
version "7.13.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.0.tgz#74424d2816f0171b4100f0ab34e9a374efdf7f80"
|
||||||
|
integrity sha512-hE+HE8rnG1Z6Wzo+MhaKE5lM5eMx71T4EHJgku2E3xIfaULhDcxiiRxUYgwX8qwP1BBSlag+TdGOt6JAidIZTA==
|
||||||
|
dependencies:
|
||||||
|
"@babel/helper-validator-identifier" "^7.12.11"
|
||||||
|
lodash "^4.17.19"
|
||||||
|
to-fast-properties "^2.0.0"
|
||||||
|
|
||||||
"@balena/dockerignore@^1.0.2":
|
"@balena/dockerignore@^1.0.2":
|
||||||
version "1.0.2"
|
version "1.0.2"
|
||||||
resolved "https://registry.yarnpkg.com/@balena/dockerignore/-/dockerignore-1.0.2.tgz#9ffe4726915251e8eb69f44ef3547e0da2c03e0d"
|
resolved "https://registry.yarnpkg.com/@balena/dockerignore/-/dockerignore-1.0.2.tgz#9ffe4726915251e8eb69f44ef3547e0da2c03e0d"
|
||||||
|
@ -1157,6 +1171,13 @@ babel-plugin-dynamic-import-node@^2.3.3:
|
||||||
dependencies:
|
dependencies:
|
||||||
object.assign "^4.1.0"
|
object.assign "^4.1.0"
|
||||||
|
|
||||||
|
babel-walk@^3.0.0:
|
||||||
|
version "3.0.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/babel-walk/-/babel-walk-3.0.0.tgz#71ea81ea1b8e2e7b37540ec8c899fe49d7fb1441"
|
||||||
|
integrity sha512-fdRxJkQ9MUSEi4jH2DcV3FAPFktk0wefilxrwNyUuWpoWawQGN7G7cB+fOYTtFfI6XNkFgwqJ/D3G18BoJJ/jg==
|
||||||
|
dependencies:
|
||||||
|
"@babel/types" "^7.9.6"
|
||||||
|
|
||||||
balanced-match@^1.0.0:
|
balanced-match@^1.0.0:
|
||||||
version "1.0.0"
|
version "1.0.0"
|
||||||
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
|
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
|
||||||
|
@ -1535,11 +1556,16 @@ commander@^2.20.0:
|
||||||
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
|
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
|
||||||
integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
|
integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
|
||||||
|
|
||||||
commander@^6.2.0, commander@^6.2.1:
|
commander@^6.2.0:
|
||||||
version "6.2.1"
|
version "6.2.1"
|
||||||
resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
|
resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
|
||||||
integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==
|
integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==
|
||||||
|
|
||||||
|
commander@^7.1.0:
|
||||||
|
version "7.1.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/commander/-/commander-7.1.0.tgz#f2eaecf131f10e36e07d894698226e36ae0eb5ff"
|
||||||
|
integrity sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg==
|
||||||
|
|
||||||
commondir@^1.0.1:
|
commondir@^1.0.1:
|
||||||
version "1.0.1"
|
version "1.0.1"
|
||||||
resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
|
resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
|
||||||
|
|
Loading…
Reference in New Issue