From 76f176a2676c30ef135a2c36ee01001e485e69ee Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 29 Jul 2020 18:33:18 -0600 Subject: [PATCH] Preliminary test runner --- backend/src/api.ts | 23 ++- backend/src/langs.ts | 22 +++ backend/src/server.ts | 2 +- backend/src/test-runner.ts | 277 +++++++++++++++++++++++++++++++++++++ backend/src/util.ts | 2 +- package.json | 3 +- 6 files changed, 324 insertions(+), 5 deletions(-) create mode 100644 backend/src/test-runner.ts diff --git a/backend/src/api.ts b/backend/src/api.ts index 5f750fd..90537f3 100644 --- a/backend/src/api.ts +++ b/backend/src/api.ts @@ -42,6 +42,8 @@ export class Session { output: string; } | null = null; + logPrimitive: (msg: string) => void; + get homedir() { return `/tmp/riju/${this.uuid}`; } @@ -62,12 +64,13 @@ export class Session { return { uid: this.uid, uuid: this.uuid }; } - log = (msg: string) => console.log(`[${this.uuid}] ${msg}`); + log = (msg: string) => this.logPrimitive(`[${this.uuid}] ${msg}`); - constructor(ws: WebSocket, lang: string) { + constructor(ws: WebSocket, lang: string, log: (msg: string) => void) { this.ws = ws; this.uuid = getUUID(); this.lang = lang; + this.logPrimitive = log; this.log(`Creating session, language ${this.lang}`); this.setup(); } @@ -248,6 +251,12 @@ export class Session { } this.lsp.writer.write(msg.input); break; + case "ensure": + if (!(this.config.test && this.config.test.ensure)) { + this.log(`ensure ignored because of missing configuration`); + break; + } + await this.ensure(); default: this.logBadMessage(msg); break; @@ -409,6 +418,16 @@ export class Session { } }; + ensure = async () => { + const code = await this.run( + this.privilegedSpawn(bash(this.config.test!.ensure!)), + { + check: false, + } + ); + this.send({ event: "ensured", code }); + }; + teardown = async () => { try { if (this.tearingDown) { diff --git a/backend/src/langs.ts b/backend/src/langs.ts index 4296922..1d8b58c 100644 --- a/backend/src/langs.ts +++ b/backend/src/langs.ts @@ -27,6 +27,28 @@ export interface LangConfig { template: string; test?: { ensure?: string; + hello?: { + pattern?: string; + }; + repl?: { + input?: string; + output?: string; + }; + scope?: { + code: string; + after?: string; + input?: string; + output?: string; + }; + format?: { + input: string; + output?: string; + }; + lsp?: { + after?: string; + code?: string; + item: string; + }; }; } diff --git a/backend/src/server.ts b/backend/src/server.ts index 862fd6f..e98c949 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -78,7 +78,7 @@ function addWebsocket( ); ws.close(); } else { - new api.Session(ws, lang); + new api.Session(ws, lang, console.log); } }); return app; diff --git a/backend/src/test-runner.ts b/backend/src/test-runner.ts new file mode 100644 index 0000000..7c29111 --- /dev/null +++ b/backend/src/test-runner.ts @@ -0,0 +1,277 @@ +import * as path from "path"; +import * as process from "process"; + +import { v4 as getUUID } from "uuid"; + +import * as api from "./api"; +import { LangConfig, langs } from "./langs"; + +const TIMEOUT_MS = 3000; + +class Test { + lang: string; + type: string; + messages: any[] = []; + timedOut: boolean = false; + handledMessages: number = 0; + handleUpdate: () => void = () => {}; + + get config() { + return langs[this.lang]; + } + + ws: any = null; + + send = (msg: any) => { + this.ws.onMessage(JSON.stringify(msg)); + }; + + constructor(lang: string, type: string) { + this.lang = lang; + this.type = type; + } + + getLog = () => { + return this.messages.map((msg: any) => JSON.stringify(msg)).join("\n"); + }; + + run = async () => { + let session = null; + try { + const that = this; + this.ws = { + on: function (type: string, handler: any) { + switch (type) { + case "message": + this.onMessage = handler; + for (const msg of this.messageQueue) { + this.onMessage(msg); + } + this.messageQueue = []; + break; + case "close": + case "error": + // No need to clean up, we'll call teardown() explicitly. + break; + default: + throw new Error(`unexpected websocket handler type: ${type}`); + } + }, + onMessage: function (msg: any) { + this.messageQueue.push(msg); + }, + messageQueue: [] as any[], + send: function (data: string) { + that.messages.push(JSON.parse(data)); + that.handleUpdate(); + }, + terminate: function () {}, + }; + session = new api.Session(this.ws, this.lang, (msg: string) => { + this.messages.push({ event: "serverLog", message: msg }); + }); + setTimeout(() => { + this.timedOut = true; + this.handleUpdate(); + }, TIMEOUT_MS); + switch (this.type) { + case "ensure": + await this.testEnsure(); + break; + case "hello": + await this.testHello(); + break; + case "repl": + await this.testRepl(); + break; + case "scope": + await this.testScope(); + break; + case "format": + await this.testFormat(); + break; + case "lsp": + await this.testLsp(); + break; + default: + throw new Error(`Unexpected test type: ${this.type}`); + } + } finally { + this.ws = null; + if (session) { + await session.teardown(); + } + } + }; + + wait = async (handler: (msg: any) => boolean) => { + return await new Promise((resolve, reject) => { + this.handleUpdate = () => { + if (this.timedOut) { + reject("timeout"); + } else { + while (this.handledMessages < this.messages.length) { + const msg = this.messages[this.handledMessages]; + const result = handler(msg); + if (result) { + resolve(result); + } + this.handledMessages += 1; + } + } + }; + this.handleUpdate(); + }); + }; + + waitForOutput = async (pattern: string) => { + let output = ""; + return await this.wait((msg: any) => { + const prevLength = output.length; + if (msg.event === "terminalOutput") { + output += msg.output; + } + return output.indexOf(pattern, prevLength - pattern.length) != -1; + }); + }; + + testEnsure = async () => { + this.send({ event: "ensure" }); + const code = await this.wait((msg: any) => { + if (msg.event === "ensured") { + return msg.code; + } + }); + if (code !== 0) { + throw new Error(`ensure failed with code ${code}`); + } + }; + testHello = async () => { + const pattern = + ((this.config.test || {}).hello || {}).pattern || "Hello, world!"; + this.send({ event: "runCode", code: this.config.template }); + await this.waitForOutput(pattern); + }; + testRepl = async () => { + const input = + ((this.config.test || {}).repl || {}).input || "111111 + 111111"; + const output = ((this.config.test || {}).repl || {}).output || "222222"; + this.send({ event: "terminalInput", input: input + "\r" }); + await this.waitForOutput(output); + }; + testRunRepl = async () => { + const input = + ((this.config.test || {}).repl || {}).input || "111111 + 111111"; + const output = ((this.config.test || {}).repl || {}).output || "222222"; + this.send({ event: "runCode", code: this.config.template }); + this.send({ event: "terminalInput", input: input + "\r" }); + await this.waitForOutput(output); + }; + testScope = async () => { + const code = this.config.test!.scope!.code; + const after = this.config.test!.scope!.after; + const input = this.config.test!.scope!.input || "x"; + const output = this.config.test!.scope!.output || "222222"; + let allCode = this.config.template; + if (!allCode.endsWith("\n")) { + allCode += "\n"; + } + if (after) { + allCode = allCode.replace(after + "\n", after + "\n" + code + "\n"); + } else { + allCode = allCode + code + "\n"; + } + this.send({ event: "runCode", code: allCode }); + this.send({ event: "terminalInput", input: input + "\r" }); + await this.waitForOutput(output); + }; + testFormat = async () => { + const input = this.config.test!.format!.input; + const output = + ((this.config.test || {}).format || { output: undefined }).output || + this.config.template; + this.send({ event: "formatCode", code: input }); + const result = await this.wait((msg: any) => { + if (msg.event === "formattedCode") { + return msg.code; + } + }); + if (output !== result) { + throw new Error("formatted code did not match"); + } + }; + testLsp = async () => {}; +} + +function lint(lang: string) { + const config = langs[lang]; + if (!config.template.endsWith("\n")) { + throw new Error("template is missing a trailing newline"); + } + if (config.format && !(config.test && config.test.format)) { + throw new Error("formatter is missing test"); + } + if (config.lsp && !(config.test && config.test.lsp)) { + throw new Error("LSP is missing test"); + } +} + +const testTypes: { + [key: string]: { + pred: (cfg: LangConfig) => boolean; + }; +} = { + ensure: { + pred: ({ test }) => (test && test.ensure ? true : false), + }, + hello: { pred: (config) => true }, + repl: { + pred: ({ repl }) => (repl ? true : false), + }, + runRepl: { + pred: ({ repl }) => (repl ? true : false), + }, + scope: { + pred: ({ test }) => (test && test.scope ? true : false), + }, + format: { + pred: ({ format }) => (format ? true : false), + }, + lsp: { pred: ({ lsp }) => (lsp ? true : false) }, +}; + +function getTestList() { + const tests: { lang: string; test: string }[] = []; + for (const [id, cfg] of Object.entries(langs)) { + for (const [test, { pred }] of Object.entries(testTypes)) { + if (pred(cfg)) { + tests.push({ lang: id, test }); + } + } + } + return tests; +} + +async function main() { + let tests = getTestList(); + const args = process.argv.slice(2); + for (const arg of args) { + tests = tests.filter(({ lang, test }) => + [lang, test].concat(langs[lang].aliases || []).includes(arg) + ); + } + for (const { lang, test: type } of tests) { + console.error(`===== LANGUAGE ${lang}, TEST ${type}`); + const test = new Test(lang, type); + try { + await test.run(); + console.error("succeeded"); + console.error(test.getLog()); + } catch (err) { + console.error("failed:"); + console.error(test.getLog()); + } + } +} + +main().catch(console.error); diff --git a/backend/src/util.ts b/backend/src/util.ts index 5e921bd..bfb9095 100644 --- a/backend/src/util.ts +++ b/backend/src/util.ts @@ -86,7 +86,7 @@ export async function run( log(`Output from ${args[0]}:\n` + output); } if (code === 0 || !check) { - resolve(); + resolve(code); } else { reject(`command ${args[0]} failed with error code ${code}`); } diff --git a/package.json b/package.json index cbe3bd9..d8f00c3 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "build": "run-s backend frontend system", "dev": "run-p backend-dev frontend-dev system-dev server-dev", "lsp-repl": "node backend/out/lsp-repl.js", - "sandbox": "node backend/out/sandbox.js" + "sandbox": "node backend/out/sandbox.js", + "test": "node backend/out/test-runner.js" } }