Effective teardown, and Sentry integration

2021-08-14 21:37:19 -07:00 · 2021-08-14 21:37:19 -07:00 · f19e850e0d
parent 46bbfa8844
commit f19e850e0d
10 changed files with 191 additions and 19 deletions
--- a/backend/api.js
+++ b/backend/api.js
@ -9,7 +9,7 @@ import rpc from "vscode-jsonrpc";

 import { langs } from "./langs.js";
 import * as util from "./util.js";
-import { bash, getUUID } from "./util.js";
+import { bash, getUUID, logError } from "./util.js";

 const allSessions = new Set();

@ -180,12 +180,11 @@ export class Session {
        await this.teardown();
      });
      this.ws.on("error", async (err) => {
-        this.log(`Websocket error: ${err}`);
+        logError(err);
        await this.teardown();
      });
    } catch (err) {
-      this.log(`Error while setting up environment`);
-      console.log(err);
+      logError(err);
      this.sendError(err);
      await this.teardown();
    }
@ -198,8 +197,7 @@ export class Session {
      }
      this.ws.send(JSON.stringify(msg));
    } catch (err) {
-      this.log(`Failed to send websocket message: ${err}`);
-      console.log(err);
+      logError(err);
      await this.teardown();
    }
  };
@ -280,8 +278,7 @@ export class Session {
          break;
      }
    } catch (err) {
-      this.log(`Error while handling message from client`);
-      console.log(err);
+      logError(err);
      this.sendError(err);
    }
  };
@ -361,8 +358,7 @@ export class Session {
        }
      });
    } catch (err) {
-      this.log(`Error while running user code`);
-      console.log(err);
+      logError(err);
      this.sendError(err);
    }
  };
@ -429,8 +425,7 @@ export class Session {
      });
      this.formatter = formatter;
    } catch (err) {
-      this.log(`Error while running code formatter`);
-      console.log(err);
+      logError(err);
      this.sendError(err);
    }
  };
@ -454,11 +449,11 @@ export class Session {
      if (this.container) {
        this.container.proc.kill();
      }
+      await this.run(this.privilegedTeardown({ uuid }));
      allSessions.delete(this);
      this.ws.terminate();
    } catch (err) {
-      this.log(`Error during teardown`);
-      console.log(err);
+      logError(err);
    }
  };
 }
--- a/backend/server.js
+++ b/backend/server.js
@ -8,7 +8,8 @@ import _ from "lodash";

 import * as api from "./api.js";
 import { aliases, langs } from "./langs.js";
-import { log } from "./util.js";
+import * as util from "./util.js";
+import { log, privilegedTeardown } from "./util.js";

 const host = process.env.HOST || "localhost";
 const port = parseInt(process.env.PORT || "") || 6119;
@ -96,6 +97,10 @@ function addWebsocket(baseApp, httpsServer) {
  return app;
 }

+util.run(privilegedTeardown(), console.error).catch((err) => {
+  console.error(err);
+});
+
 if (useTLS) {
  const httpsServer = https.createServer(
    {
--- a/backend/util.js
+++ b/backend/util.js
@ -4,6 +4,22 @@ import process from "process";

 import { v4 as getUUIDOrig } from "uuid";

+let sentryEnabled = false;
+
+if (process.env.SENTRY_DSN) {
+  Sentry.init({
+    dsn: process.env.SENTRY_DSN,
+  });
+  sentryEnabled = true;
+}
+
+export function logError(err) {
+  console.error(err);
+  if (sentryEnabled) {
+    Sentry.captureException(err);
+  }
+}
+
 function computeImageHashes() {
  let deployConfig = process.env.RIJU_DEPLOY_CONFIG;
  if (!deployConfig) return {};
@ -85,6 +101,16 @@ export function privilegedPty({ uuid }, args) {
  return [rijuSystemPrivileged, "pty", uuid].concat(args);
 }

+export function privilegedTeardown(options) {
+  options = options || {};
+  const { uuid } = options;
+  const cmdline = [rijuSystemPrivileged, "teardown"];
+  if (uuid) {
+    cmdline.push(uuid);
+  }
+  return cmdline;
+}
+
 export function bash(cmdline, opts) {
  const stty = opts && opts.stty;
  if (!cmdline.match(/[;|&(){}=\n]/)) {
--- a/package.json
+++ b/package.json
@ -9,6 +9,7 @@
    "@babel/parser": "^7.13.11",
    "@babel/preset-env": "^7.12.11",
    "@balena/dockerignore": "^1.0.2",
+    "@sentry/node": "^6.11.0",
    "async-lock": "^1.2.6",
    "babel-loader": "^8.2.2",
    "babel-walk": "^3.0.0",
--- a/packer/provision-web.bash
+++ b/packer/provision-web.bash
@ -61,6 +61,7 @@ sudo sed -Ei 's/^#?PermitEmptyPasswords .*/PermitEmptyPasswords no/' /etc/ssh/ss
 sudo sed -Ei "s/\\\$AWS_REGION/${AWS_REGION}/" /etc/systemd/system/riju.service
 sudo sed -Ei "s/\\\$FATHOM_SITE_ID/${FATHOM_SITE_ID:-}/" /etc/systemd/system/riju.service
 sudo sed -Ei "s/\\\$S3_BUCKET/${S3_BUCKET}/" /etc/systemd/system/riju.service
+sudo sed -Ei "s/\\\$SENTRY_DSN/${SENTRY_DSN:-}/" /etc/systemd/system/riju.service
 sudo sed -Ei "s/\\\$SUPERVISOR_ACCESS_TOKEN/${SUPERVISOR_ACCESS_TOKEN}/" /etc/systemd/system/riju.service

 sudo passwd -l root
--- a/packer/riju.service
+++ b/packer/riju.service
@ -13,6 +13,7 @@ RestartSec=5
 Environment=AWS_REGION=$AWS_REGION
 Environment=FATHOM_SITE_ID=$FATHOM_SITE_ID
 Environment=S3_BUCKET=$S3_BUCKET
+Environment=SENTRY_DSN=$SENTRY_DSN
 Environment=SUPERVISOR_ACCESS_TOKEN=$SUPERVISOR_ACCESS_TOKEN

 [Install]
--- a/packer/web.pkr.hcl
+++ b/packer/web.pkr.hcl
@ -23,6 +23,11 @@ variable "s3_bucket" {
  default = "${env("S3_BUCKET")}"
 }

+variable "sentry_dsn" {
+  type = string
+  default = "${env("SENTRY_DSN_PACKER")}"
+}
+
 variable "supervisor_access_token" {
  type = string
  default = "${env("SUPERVISOR_ACCESS_TOKEN")}"
@ -114,6 +119,7 @@ build {
      "FATHOM_SITE_ID=${var.fathom_site_id}",
      "GRAFANA_API_KEY=${var.grafana_api_key}",
      "S3_BUCKET=${var.s3_bucket}",
+      "SENTRY_DSN=${var.sentry_dsn}",
      "SUPERVISOR_ACCESS_TOKEN=${var.supervisor_access_token}",
    ]
    script           = "provision-web.bash"
--- a/supervisor/src/main.go
+++ b/supervisor/src/main.go
@ -349,6 +349,7 @@ func (sv *supervisor) reload() error {
 		"-p", fmt.Sprintf("127.0.0.1:%d:6119", port),
 		"-e", "FATHOM_SITE_ID",
 		"-e", "RIJU_DEPLOY_CONFIG",
+		"-e", "SENTRY_DSN",
 		"--label", fmt.Sprintf("riju.deploy-config-hash=%s", deployCfgHash),
 		"--name", name,
 		"--restart", "unless-stopped",
--- a/system/src/riju-system-privileged.c
+++ b/system/src/riju-system-privileged.c
@ -32,7 +32,8 @@ void die_with_usage()
  die("usage:\n"
      "  riju-system-privileged session UUID LANG [IMAGE-HASH]\n"
      "  riju-system-privileged exec UUID CMDLINE...\n"
-      "  riju-system-privileged pty UUID CMDLINE...");
+      "  riju-system-privileged pty UUID CMDLINE...\n"
+      "  riju-system-privileged teardown [UUID]");
 }

 char *quoteArgs(int argc, char **cmdline)
@ -128,7 +129,10 @@ void session(char *uuid, char *lang, char *imageHash)
 {
  if (setvbuf(stdout, NULL, _IONBF, 0) != 0)
    die("setvbuf failed");
-  char *image, *container, *hostname, *share, *volume, *fifo, *rijuPtyPath;
+  char *image, *container, *hostname, *share, *volume, *fifo, *rijuPtyPath,
+      *sessionLabel;
+  if (asprintf(&sessionLabel, "riju.user-session=%s", uuid) < 0)
+    die("asprintf failed");
  if ((imageHash != NULL ? asprintf(&image, "riju:lang-%s-%s", lang, imageHash)
                         : asprintf(&image, "riju:lang-%s", lang)) < 0)
    die("asprintf failed");
@ -231,6 +235,10 @@ void session(char *uuid, char *lang, char *imageHash)
        "2048",
        "--cgroup-parent",
        "riju.slice",
+        "--label",
+        "riju.category=user-session",
+        "--label",
+        sessionLabel,
        image,
        "bash",
        "-c",
@ -403,11 +411,28 @@ void exec(char *uuid, int argc, char **cmdline, bool pty)
  }
 }

+void teardown(char *uuid)
+{
+  char *cmdline;
+  if (uuid != NULL) {
+    if (asprintf(&cmdline, "rm -rf /var/cache/riju/shares/%s", uuid) < 0)
+      die("asprintf failed");
+  } else {
+    cmdline = "comm -23 <(sudo ls /var/cache/riju/shares | sort) <(docker ps "
+              "-f label=riju.category=user-session --format \"{{ .Labels }}\" "
+              "| grep -Eo 'riju\\.user-session=[a-z0-9]+' | sed -E "
+              "'s/^[^=]+=//') | (cd /var/cache/riju/shares; xargs rm -rf)";
+  }
+  char *argv[] = {"bash", "-c", cmdline, NULL};
+  execvp(argv[0], argv);
+  die("execvp failed");
+}
+
 int main(int argc, char **argv)
 {
  init();
-  if (seteuid(0) != 0)
-    die("seteuid failed");
+  if (setuid(0) != 0)
+    die("setuid failed");
  if (argc < 2)
    die_with_usage();
  if (!strcmp(argv[1], "session")) {
@ -431,5 +456,11 @@ int main(int argc, char **argv)
    exec(parseUUID(argv[2]), argc - 3, &argv[3], true);
    return 0;
  }
+  if (!strcmp(argv[1], "teardown")) {
+    if (argc < 2)
+      die_with_usage();
+    teardown(argc >= 3 ? parseUUID(argv[2]) : NULL);
+    return 0;
+  }
  die_with_usage();
 }
--- a/yarn.lock
+++ b/yarn.lock
@ -878,6 +878,74 @@
  resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.3.tgz#90420f9f9c6d3987f176a19a7d8e764271a2f55d"
  integrity sha512-Fxt+AfXgjMoin2maPIYzFZnQjAXjAL0PHscM5pRTtatFqB+vZxAM9tLp2Optnuw3QOQC40jTNeGYFOMvyf7v9g==

+"@sentry/core@6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/core/-/core-6.11.0.tgz#40e94043afcf6407a109be26655c77832c64e740"
+  integrity sha512-09TB+f3pqEq8LFahFWHO6I/4DxHo+NcS52OkbWMDqEi6oNZRD7PhPn3i14LfjsYVv3u3AESU8oxSEGbFrr2UjQ==
+  dependencies:
+    "@sentry/hub" "6.11.0"
+    "@sentry/minimal" "6.11.0"
+    "@sentry/types" "6.11.0"
+    "@sentry/utils" "6.11.0"
+    tslib "^1.9.3"
+
+"@sentry/hub@6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/hub/-/hub-6.11.0.tgz#ddf9ddb0577d1c8290dc02c0242d274fe84d6c16"
+  integrity sha512-pT9hf+ZJfVFpoZopoC+yJmFNclr4NPqPcl2cgguqCHb69DklD1NxgBNWK8D6X05qjnNFDF991U6t1mxP9HrGuw==
+  dependencies:
+    "@sentry/types" "6.11.0"
+    "@sentry/utils" "6.11.0"
+    tslib "^1.9.3"
+
+"@sentry/minimal@6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/minimal/-/minimal-6.11.0.tgz#806d5512658370e40827b3e3663061db708fff33"
+  integrity sha512-XkZ7qrdlGp4IM/gjGxf1Q575yIbl5RvPbg+WFeekpo16Ufvzx37Mr8c2xsZaWosISVyE6eyFpooORjUlzy8EDw==
+  dependencies:
+    "@sentry/hub" "6.11.0"
+    "@sentry/types" "6.11.0"
+    tslib "^1.9.3"
+
+"@sentry/node@^6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/node/-/node-6.11.0.tgz#62614c18af779373a12311f2fb57a8dde278425a"
+  integrity sha512-vbk+V/n7ZIFD8rHPYy03t/gIG5V7LGdjU4qJxVDgNZzticfWPnd2sLgle/r+l60XF6SKW/epG4rnxnBcgPdWaw==
+  dependencies:
+    "@sentry/core" "6.11.0"
+    "@sentry/hub" "6.11.0"
+    "@sentry/tracing" "6.11.0"
+    "@sentry/types" "6.11.0"
+    "@sentry/utils" "6.11.0"
+    cookie "^0.4.1"
+    https-proxy-agent "^5.0.0"
+    lru_map "^0.3.3"
+    tslib "^1.9.3"
+
+"@sentry/tracing@6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/tracing/-/tracing-6.11.0.tgz#9bd9287addea1ebc12c75b226f71c7713c0fac4f"
+  integrity sha512-9VA1/SY++WeoMQI4K6n/sYgIdRtCu9NLWqmGqu/5kbOtESYFgAt1DqSyqGCr00ZjQiC2s7tkDkTNZb38K6KytQ==
+  dependencies:
+    "@sentry/hub" "6.11.0"
+    "@sentry/minimal" "6.11.0"
+    "@sentry/types" "6.11.0"
+    "@sentry/utils" "6.11.0"
+    tslib "^1.9.3"
+
+"@sentry/types@6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/types/-/types-6.11.0.tgz#5122685478d32ddacd3a891cbcf550012df85f7c"
+  integrity sha512-gm5H9eZhL6bsIy/h3T+/Fzzz2vINhHhqd92CjHle3w7uXdTdFV98i2pDpErBGNTSNzbntqOMifYEB5ENtZAvcg==
+
+"@sentry/utils@6.11.0":
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-6.11.0.tgz#d1dee4faf4d9c42c54bba88d5a66fb96b902a14c"
+  integrity sha512-IOvyFHcnbRQxa++jO+ZUzRvFHEJ1cZjrBIQaNVc0IYF0twUOB5PTP6joTcix38ldaLeapaPZ9LGfudbvYvxkdg==
+  dependencies:
+    "@sentry/types" "6.11.0"
+    tslib "^1.9.3"
+
 "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6":
  version "7.0.7"
  resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad"
@ -1068,6 +1136,13 @@ acorn@^6.4.1:
  resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6"
  integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==

+agent-base@6:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
+  integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
+  dependencies:
+    debug "4"
+
 ajv-errors@^1.0.0:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d"
@ -1691,6 +1766,11 @@ cookie@0.4.0:
  resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba"
  integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==

+cookie@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1"
+  integrity sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==
+
 copy-concurrently@^1.0.0:
  version "1.0.5"
  resolved "https://registry.yarnpkg.com/copy-concurrently/-/copy-concurrently-1.0.5.tgz#92297398cae34937fcafd6ec8139c18051f0b5e0"
@ -1816,6 +1896,13 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3:
  dependencies:
    ms "2.0.0"

+debug@4:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
+  integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
+  dependencies:
+    ms "2.1.2"
+
 debug@^4.1.0, debug@^4.1.1:
  version "4.3.1"
  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
@ -2498,6 +2585,14 @@ https-browserify@^1.0.0:
  resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"
  integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=

+https-proxy-agent@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
+  integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==
+  dependencies:
+    agent-base "6"
+    debug "4"
+
 human-signals@^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
@ -2882,6 +2977,11 @@ lru-cache@^6.0.0:
  dependencies:
    yallist "^4.0.0"

+lru_map@^0.3.3:
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd"
+  integrity sha1-tcg1G5Rky9dQM1p5ZQoOwOVhGN0=
+
 make-dir@^2.0.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
@ -4202,6 +4302,11 @@ toidentifier@1.0.0:
  resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553"
  integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==

+tslib@^1.9.3:
+  version "1.14.1"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
+  integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
+
 tty-browserify@0.0.0:
  version "0.0.0"
  resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"