Improve large-scale test result reporting

.gitignore

@@ -3,3 +3,4 @@
 .lsp-repl-history
 node_modules
 out
+tests

backend/src/test-runner.ts

@@ -1,12 +1,16 @@
-import * as path from "path";
+import * as fs from "fs";
 import * as process from "process";
+import { promisify } from "util";
 
+import PQueue from "p-queue";
+import * as rimraf from "rimraf";
 import { v4 as getUUID } from "uuid";
 
 import * as api from "./api";
 import { LangConfig, langs } from "./langs";
 
 const TIMEOUT_MS = 3000;
+const CONCURRENCY = 16;
 
 function findPosition(str: string, idx: number) {
   const lines = str.substring(0, idx).split("\n");
@@ -40,8 +44,11 @@ class Test {
     this.type = type;
   }
 
-  getLog = () => {
+  getLog = (opts?: any) => {
-    return this.messages.map((msg: any) => JSON.stringify(msg)).join("\n");
+    opts = opts || {};
+    return this.messages
+      .map((msg: any) => JSON.stringify(msg, null, opts.pretty && 2))
+      .join("\n");
   };
 
   run = async () => {
@@ -508,6 +515,14 @@ function lint(lang: string) {
   if (!config.template.endsWith("\n")) {
     throw new Error("template is missing a trailing newline");
   }
+  // These can be removed when the types are adjusted to make these
+  // situations impossible.
+  if (config.format && !config.format.input) {
+    throw new Error("formatter is missing test");
+  }
+  if (config.lsp && !(config.lsp.code && config.lsp.item)) {
+    throw new Error("LSP is missing test");
+  }
 }
 
 const testTypes: {
@@ -546,6 +561,11 @@ function getTestList() {
   return tests;
 }
 
+async function writeLog(lang: string, type: string, log: string) {
+  await promisify(fs.mkdir)(`tests/${lang}`, { recursive: true });
+  await promisify(fs.writeFile)(`tests/${lang}/${type}.log`, log);
+}
+
 async function main() {
   let tests = getTestList();
   const args = process.argv.slice(2);
@@ -564,42 +584,68 @@ async function main() {
     process.exit(1);
   }
   const lintSeen = new Set();
-  let lintPassed = 0;
+  let lintPassed = new Set();
-  let lintFailed = 0;
+  let lintFailed = new Map();
   for (const { lang } of tests) {
     if (!lintSeen.has(lang)) {
       lintSeen.add(lang);
-      console.error(`===== LANGUAGE ${lang}, LINT`);
       try {
         lint(lang);
-        console.error("passed");
+        lintPassed.add(lang);
-        lintPassed += 1;
       } catch (err) {
-        console.error("failed");
+        lintFailed.set(lang, err);
-        console.error(err);
-        lintFailed += 1;
       }
     }
   }
-  if (lintFailed) {
+  if (lintFailed.size > 0) {
+    console.error(
+      `Language${lintFailed.size !== 1 ? "s" : ""} failed linting:`
+    );
+    console.error(
+      Array.from(lintFailed)
+        .map(([lang, err]) => `  - ${lang} (${err})`)
+        .join("\n")
+    );
     process.exit(1);
   }
+  await promisify(rimraf)("tests");
+  const queue = new PQueue({ concurrency: CONCURRENCY });
   let passed = 0;
   let failed = 0;
   for (const { lang, test: type } of tests) {
-    console.error(`===== LANGUAGE ${lang}, TEST ${type}`);
+    let test: Test;
-    const test = new Test(lang, type);
+    queue
-    try {
+      .add(() => {
-      await test.run();
+        test = new Test(lang, type);
-      console.error("passed");
+        return test.run();
+      })
+      .then(async () => {
         passed += 1;
-    } catch (err) {
+        console.error(`PASSED: ${lang}/${type}`);
-      console.error("failed");
+        await writeLog(
+          lang,
+          type,
+          `PASSED: ${lang}/${type}\n` + test.getLog({ pretty: true }) + "\n"
+        );
+      })
+      .catch(async (err) => {
+        failed += 1;
+        console.error(`FAILED: ${lang}/${type}`);
         console.error(test.getLog());
         console.error(err);
-      failed += 1;
+        await writeLog(
-    }
+          lang,
+          type,
+          `FAILED: ${lang}/${type}\n` +
+            test.getLog({ pretty: true }) +
+            "\n" +
+            err.stack +
+            "\n"
+        );
+      })
+      .catch(console.error);
   }
+  await queue.onIdle();
   process.exit(failed ? 1 : 0);
 }
 

package.json

@@ -35,6 +35,7 @@
     "npm-run-all": "^4.1.5",
     "p-queue": "^6.6.0",
     "parse-passwd": "^1.0.0",
+    "rimraf": "^3.0.2",
     "shell-quote": "^1.7.2",
     "style-loader": "^1.2.1",
     "ts-loader": "^7.0.5",

yarn.lock

@@ -4189,6 +4189,13 @@ rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.3:
   dependencies:
     glob "^7.1.3"
 
+rimraf@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
+  integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==
+  dependencies:
+    glob "^7.1.3"
+
 ripemd160@^2.0.0, ripemd160@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c"