Preliminary test runner

backend/src/api.ts

@@ -42,6 +42,8 @@ export class Session {
     output: string;
   } | null = null;
 
+  logPrimitive: (msg: string) => void;
+
   get homedir() {
     return `/tmp/riju/${this.uuid}`;
   }
@@ -62,12 +64,13 @@ export class Session {
     return { uid: this.uid, uuid: this.uuid };
   }
 
-  log = (msg: string) => console.log(`[${this.uuid}] ${msg}`);
+  log = (msg: string) => this.logPrimitive(`[${this.uuid}] ${msg}`);
 
-  constructor(ws: WebSocket, lang: string) {
+  constructor(ws: WebSocket, lang: string, log: (msg: string) => void) {
     this.ws = ws;
     this.uuid = getUUID();
     this.lang = lang;
+    this.logPrimitive = log;
     this.log(`Creating session, language ${this.lang}`);
     this.setup();
   }
@@ -248,6 +251,12 @@ export class Session {
           }
           this.lsp.writer.write(msg.input);
           break;
+        case "ensure":
+          if (!(this.config.test && this.config.test.ensure)) {
+            this.log(`ensure ignored because of missing configuration`);
+            break;
+          }
+          await this.ensure();
         default:
           this.logBadMessage(msg);
           break;
@@ -409,6 +418,16 @@ export class Session {
     }
   };
 
+  ensure = async () => {
+    const code = await this.run(
+      this.privilegedSpawn(bash(this.config.test!.ensure!)),
+      {
+        check: false,
+      }
+    );
+    this.send({ event: "ensured", code });
+  };
+
   teardown = async () => {
     try {
       if (this.tearingDown) {

backend/src/langs.ts

@@ -27,6 +27,28 @@ export interface LangConfig {
   template: string;
   test?: {
     ensure?: string;
+    hello?: {
+      pattern?: string;
+    };
+    repl?: {
+      input?: string;
+      output?: string;
+    };
+    scope?: {
+      code: string;
+      after?: string;
+      input?: string;
+      output?: string;
+    };
+    format?: {
+      input: string;
+      output?: string;
+    };
+    lsp?: {
+      after?: string;
+      code?: string;
+      item: string;
+    };
   };
 }
 

backend/src/server.ts

@@ -78,7 +78,7 @@ function addWebsocket(
       );
       ws.close();
     } else {
-      new api.Session(ws, lang);
+      new api.Session(ws, lang, console.log);
     }
   });
   return app;

backend/src/test-runner.ts

@@ -0,0 +1,277 @@
+import * as path from "path";
+import * as process from "process";
+
+import { v4 as getUUID } from "uuid";
+
+import * as api from "./api";
+import { LangConfig, langs } from "./langs";
+
+const TIMEOUT_MS = 3000;
+
+class Test {
+  lang: string;
+  type: string;
+  messages: any[] = [];
+  timedOut: boolean = false;
+  handledMessages: number = 0;
+  handleUpdate: () => void = () => {};
+
+  get config() {
+    return langs[this.lang];
+  }
+
+  ws: any = null;
+
+  send = (msg: any) => {
+    this.ws.onMessage(JSON.stringify(msg));
+  };
+
+  constructor(lang: string, type: string) {
+    this.lang = lang;
+    this.type = type;
+  }
+
+  getLog = () => {
+    return this.messages.map((msg: any) => JSON.stringify(msg)).join("\n");
+  };
+
+  run = async () => {
+    let session = null;
+    try {
+      const that = this;
+      this.ws = {
+        on: function (type: string, handler: any) {
+          switch (type) {
+            case "message":
+              this.onMessage = handler;
+              for (const msg of this.messageQueue) {
+                this.onMessage(msg);
+              }
+              this.messageQueue = [];
+              break;
+            case "close":
+            case "error":
+              // No need to clean up, we'll call teardown() explicitly.
+              break;
+            default:
+              throw new Error(`unexpected websocket handler type: ${type}`);
+          }
+        },
+        onMessage: function (msg: any) {
+          this.messageQueue.push(msg);
+        },
+        messageQueue: [] as any[],
+        send: function (data: string) {
+          that.messages.push(JSON.parse(data));
+          that.handleUpdate();
+        },
+        terminate: function () {},
+      };
+      session = new api.Session(this.ws, this.lang, (msg: string) => {
+        this.messages.push({ event: "serverLog", message: msg });
+      });
+      setTimeout(() => {
+        this.timedOut = true;
+        this.handleUpdate();
+      }, TIMEOUT_MS);
+      switch (this.type) {
+        case "ensure":
+          await this.testEnsure();
+          break;
+        case "hello":
+          await this.testHello();
+          break;
+        case "repl":
+          await this.testRepl();
+          break;
+        case "scope":
+          await this.testScope();
+          break;
+        case "format":
+          await this.testFormat();
+          break;
+        case "lsp":
+          await this.testLsp();
+          break;
+        default:
+          throw new Error(`Unexpected test type: ${this.type}`);
+      }
+    } finally {
+      this.ws = null;
+      if (session) {
+        await session.teardown();
+      }
+    }
+  };
+
+  wait = async (handler: (msg: any) => boolean) => {
+    return await new Promise((resolve, reject) => {
+      this.handleUpdate = () => {
+        if (this.timedOut) {
+          reject("timeout");
+        } else {
+          while (this.handledMessages < this.messages.length) {
+            const msg = this.messages[this.handledMessages];
+            const result = handler(msg);
+            if (result) {
+              resolve(result);
+            }
+            this.handledMessages += 1;
+          }
+        }
+      };
+      this.handleUpdate();
+    });
+  };
+
+  waitForOutput = async (pattern: string) => {
+    let output = "";
+    return await this.wait((msg: any) => {
+      const prevLength = output.length;
+      if (msg.event === "terminalOutput") {
+        output += msg.output;
+      }
+      return output.indexOf(pattern, prevLength - pattern.length) != -1;
+    });
+  };
+
+  testEnsure = async () => {
+    this.send({ event: "ensure" });
+    const code = await this.wait((msg: any) => {
+      if (msg.event === "ensured") {
+        return msg.code;
+      }
+    });
+    if (code !== 0) {
+      throw new Error(`ensure failed with code ${code}`);
+    }
+  };
+  testHello = async () => {
+    const pattern =
+      ((this.config.test || {}).hello || {}).pattern || "Hello, world!";
+    this.send({ event: "runCode", code: this.config.template });
+    await this.waitForOutput(pattern);
+  };
+  testRepl = async () => {
+    const input =
+      ((this.config.test || {}).repl || {}).input || "111111 + 111111";
+    const output = ((this.config.test || {}).repl || {}).output || "222222";
+    this.send({ event: "terminalInput", input: input + "\r" });
+    await this.waitForOutput(output);
+  };
+  testRunRepl = async () => {
+    const input =
+      ((this.config.test || {}).repl || {}).input || "111111 + 111111";
+    const output = ((this.config.test || {}).repl || {}).output || "222222";
+    this.send({ event: "runCode", code: this.config.template });
+    this.send({ event: "terminalInput", input: input + "\r" });
+    await this.waitForOutput(output);
+  };
+  testScope = async () => {
+    const code = this.config.test!.scope!.code;
+    const after = this.config.test!.scope!.after;
+    const input = this.config.test!.scope!.input || "x";
+    const output = this.config.test!.scope!.output || "222222";
+    let allCode = this.config.template;
+    if (!allCode.endsWith("\n")) {
+      allCode += "\n";
+    }
+    if (after) {
+      allCode = allCode.replace(after + "\n", after + "\n" + code + "\n");
+    } else {
+      allCode = allCode + code + "\n";
+    }
+    this.send({ event: "runCode", code: allCode });
+    this.send({ event: "terminalInput", input: input + "\r" });
+    await this.waitForOutput(output);
+  };
+  testFormat = async () => {
+    const input = this.config.test!.format!.input;
+    const output =
+      ((this.config.test || {}).format || { output: undefined }).output ||
+      this.config.template;
+    this.send({ event: "formatCode", code: input });
+    const result = await this.wait((msg: any) => {
+      if (msg.event === "formattedCode") {
+        return msg.code;
+      }
+    });
+    if (output !== result) {
+      throw new Error("formatted code did not match");
+    }
+  };
+  testLsp = async () => {};
+}
+
+function lint(lang: string) {
+  const config = langs[lang];
+  if (!config.template.endsWith("\n")) {
+    throw new Error("template is missing a trailing newline");
+  }
+  if (config.format && !(config.test && config.test.format)) {
+    throw new Error("formatter is missing test");
+  }
+  if (config.lsp && !(config.test && config.test.lsp)) {
+    throw new Error("LSP is missing test");
+  }
+}
+
+const testTypes: {
+  [key: string]: {
+    pred: (cfg: LangConfig) => boolean;
+  };
+} = {
+  ensure: {
+    pred: ({ test }) => (test && test.ensure ? true : false),
+  },
+  hello: { pred: (config) => true },
+  repl: {
+    pred: ({ repl }) => (repl ? true : false),
+  },
+  runRepl: {
+    pred: ({ repl }) => (repl ? true : false),
+  },
+  scope: {
+    pred: ({ test }) => (test && test.scope ? true : false),
+  },
+  format: {
+    pred: ({ format }) => (format ? true : false),
+  },
+  lsp: { pred: ({ lsp }) => (lsp ? true : false) },
+};
+
+function getTestList() {
+  const tests: { lang: string; test: string }[] = [];
+  for (const [id, cfg] of Object.entries(langs)) {
+    for (const [test, { pred }] of Object.entries(testTypes)) {
+      if (pred(cfg)) {
+        tests.push({ lang: id, test });
+      }
+    }
+  }
+  return tests;
+}
+
+async function main() {
+  let tests = getTestList();
+  const args = process.argv.slice(2);
+  for (const arg of args) {
+    tests = tests.filter(({ lang, test }) =>
+      [lang, test].concat(langs[lang].aliases || []).includes(arg)
+    );
+  }
+  for (const { lang, test: type } of tests) {
+    console.error(`===== LANGUAGE ${lang}, TEST ${type}`);
+    const test = new Test(lang, type);
+    try {
+      await test.run();
+      console.error("succeeded");
+      console.error(test.getLog());
+    } catch (err) {
+      console.error("failed:");
+      console.error(test.getLog());
+    }
+  }
+}
+
+main().catch(console.error);

backend/src/util.ts

@@ -86,7 +86,7 @@ export async function run(
         log(`Output from ${args[0]}:\n` + output);
       }
       if (code === 0 || !check) {
-        resolve();
+        resolve(code);
       } else {
         reject(`command ${args[0]} failed with error code ${code}`);
       }

package.json

@@ -58,6 +58,7 @@
     "build": "run-s backend frontend system",
     "dev": "run-p backend-dev frontend-dev system-dev server-dev",
     "lsp-repl": "node backend/out/lsp-repl.js",
-    "sandbox": "node backend/out/sandbox.js"
+    "sandbox": "node backend/out/sandbox.js",
+    "test": "node backend/out/test-runner.js"
   }
 }