Add/change various things

agent/go.mod

@@ -2,4 +2,7 @@ module github.com/radian-software/riju/agent
 
 go 1.18
 
-require github.com/gorilla/websocket v1.5.0 // indirect
+require (
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+)

agent/go.sum

@@ -1,2 +1,4 @@
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

agent/main.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"time"
 
+	"github.com/google/shlex"
 	"github.com/gorilla/websocket"
 )
 
@@ -67,6 +68,25 @@ func warnf(ms *ManagedWebsocket, format string, arg ...interface{}) {
 	warn(ms, fmt.Errorf(format, arg...))
 }
 
+func getCommandPrefix() []string {
+	prefix := os.Getenv("RIJU_AGENT_COMMAND_PREFIX")
+	if prefix == "" {
+		logErrorf("must specify RIJU_AGENT_COMMAND_PREFIX for security reasons")
+		os.Exit(1)
+	}
+	if prefix == "0" {
+		return []string{}
+	}
+	list, err := shlex.Split(prefix)
+	if err != nil {
+		logErrorf("parsing RIJU_AGENT_COMMAND_PREFIX: %w", err)
+		os.Exit(1)
+	}
+	return list
+}
+
+var CommandPrefix = getCommandPrefix()
+
 // https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go
 func handler(w http.ResponseWriter, r *http.Request) {
 	// Upgrade http connection to websocket
@@ -98,6 +118,7 @@ func handler(w http.ResponseWriter, r *http.Request) {
 		fatalf(ms, "cmdline query parameter missing")
 		return
 	}
+	cmdline = append(CommandPrefix, cmdline...)
 	binary, err := exec.LookPath(cmdline[0])
 	if err != nil {
 		fatalf(ms, "searching for executable: %w", err)
@@ -191,7 +212,12 @@ func main() {
 		host = "0.0.0.0"
 	}
 	fmt.Printf("Listening on http://%s:%s\n", host, port)
-	err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), http.HandlerFunc(handler))
+	mux := http.NewServeMux()
+	mux.HandleFunc("/exec", handler)
+	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+	err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), mux)
 	if err != nil {
 		logError(err)
 		os.Exit(1)

backend/k8s.js

@@ -1,10 +1,54 @@
 import * as k8sClient from "@kubernetes/client-node";
+import lodash from "lodash";
 
 const kubeconfig = new k8sClient.KubeConfig();
 kubeconfig.loadFromDefault();
 
 const k8s = kubeconfig.makeApiClient(k8sClient.CoreV1Api);
 
+export function watchPods() {
+  const callbacks = {};
+  const pods = {};
+
+  // https://github.com/kubernetes-client/javascript/blob/1f76ee10c54e33a998abb4686488ccff4285366a/examples/typescript/informer/informer.ts
+  //
+  // The watch functionality seems to be wholly undocumented. Copy,
+  // paste, and pray.
+  const informer = k8sClient.makeInformer(
+    kubeconfig,
+    "/api/v1/namespaces/riju-user/pods",
+    () => k8s.listNamespacedPod("riju-user")
+  );
+
+  for (const event of ["add", "update", "delete"]) {
+    informer.on(event, (pod) => {
+      if (pod.metadata.name in callbacks) {
+        callbacks[pod.metadata.name](event, pod);
+      }
+      pods[pod.metadata.name] = pod;
+      if (event == "delete") {
+        delete callbacks[pod.metadata.name];
+        delete pods[pod.metadata.name];
+      }
+    });
+  }
+
+  informer.on("error", (err) => {
+    console.error(err);
+    setTimeout(() => informer.start(), 5000);
+  });
+  informer.start();
+
+  return {
+    setCallback: (podName, callback) => {
+      callbacks[podName] = callback;
+      if (podName in pods) {
+        callback("add", pods[podName]);
+      }
+    },
+  };
+}
+
 export async function listUserSessions() {
   return (await k8s.listNamespacedPod("riju-user")).body.items.map((pod) => ({
     podName: pod.metadata.name,
@@ -12,116 +56,150 @@ export async function listUserSessions() {
   }));
 }
 
-export async function createUserSession({ sessionID, langConfig, revisions }) {
+export async function createUserSession({
-  const { body: pod } = await k8s.createNamespacedPod("riju-user", {
+  watcher,
-    metadata: {
+  sessionID,
-      name: `riju-user-session-${sessionID}`,
+  langConfig,
-      labels: {
+  revisions,
-        "riju.codes/user-session-id": sessionID,
+}) {
+  const pod = (
+    await k8s.createNamespacedPod("riju-user", {
+      metadata: {
+        name: `riju-user-session-${sessionID}`,
+        labels: {
+          "riju.codes/user-session-id": sessionID,
+        },
       },
-    },
+      spec: {
-    spec: {
+        volumes: [
-      volumes: [
+          {
-        {
+            name: "minio-config",
-          name: "minio-config",
+            secret: {
-          secret: {
+              secretName: "minio-user-login",
-            secretName: "minio-user-login",
-          },
-        },
-        {
-          name: "riju-bin",
-          emptyDir: {},
-        },
-      ],
-      imagePullSecrets: [
-        {
-          name: "registry-user-login",
-        },
-      ],
-      initContainers: [
-        {
-          name: "download",
-          image: "minio/mc:RELEASE.2022-12-13T00-23-28Z",
-          resources: {},
-          command: ["sh", "-c"],
-          args: [
-            `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` +
-              `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` +
-              `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`,
-          ],
-          volumeMounts: [
-            {
-              name: "minio-config",
-              mountPath: "/mc",
-              readOnly: true,
-            },
-            {
-              name: "riju-bin",
-              mountPath: "/riju-bin",
-            },
-          ],
-        },
-      ],
-      containers: [
-        {
-          name: "session",
-          image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`,
-          resources: {
-            requests: {},
-            limits: {
-              cpu: "1000m",
-              memory: "4Gi",
             },
           },
-          startupProbe: {
+          {
-            httpGet: {
+            name: "riju-bin",
-              path: "/health",
+            emptyDir: {},
-              port: 869,
-              scheme: "HTTP",
-            },
-            failureThreshold: 30,
-            initialDelaySeconds: 0,
-            periodSeconds: 1,
-            successThreshold: 1,
-            timeoutSeconds: 2,
           },
-          readinessProbe: {
+        ],
-            httpGet: {
+        imagePullSecrets: [
-              path: "/health",
+          {
-              port: 869,
+            name: "registry-user-login",
-              scheme: "HTTP",
-            },
-            failureThreshold: 1,
-            initialDelaySeconds: 2,
-            periodSeconds: 10,
-            successThreshold: 1,
-            timeoutSeconds: 2,
           },
-          livenessProbe: {
+        ],
-            httpGet: {
+        initContainers: [
-              path: "/health",
+          {
-              port: 869,
+            name: "download",
-              scheme: "HTTP",
+            image: "minio/mc:RELEASE.2022-12-13T00-23-28Z",
-            },
+            resources: {},
-            failureThreshold: 3,
+            command: ["sh", "-c"],
-            initialDelaySeconds: 2,
+            args: [
-            periodSeconds: 10,
+              `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` +
-            successThreshold: 1,
+                `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` +
-            timeoutSeconds: 2,
+                `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`,
+            ],
+            volumeMounts: [
+              {
+                name: "minio-config",
+                mountPath: "/mc",
+                readOnly: true,
+              },
+              {
+                name: "riju-bin",
+                mountPath: "/riju-bin",
+              },
+            ],
           },
-          volumeMounts: [
+        ],
-            {
+        containers: [
-              name: "riju-bin",
+          {
-              mountPath: "/riju-bin",
+            name: "session",
-              readOnly: true,
+            image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`,
+            resources: {
+              requests: {},
+              limits: {
+                cpu: "1000m",
+                memory: "4Gi",
+              },
             },
-          ],
+            command: ["/riju-bin/agent"],
-        },
+            env: [
-      ],
+              {
-      restartPolicy: "Never",
+                name: "RIJU_AGENT_COMMAND_PREFIX",
-    },
+                value: "runuser -u riju --",
+              },
+            ],
+            securityContext: {
+              runAsUser: 0,
+            },
+            startupProbe: {
+              httpGet: {
+                path: "/health",
+                port: 869,
+                scheme: "HTTP",
+              },
+              failureThreshold: 30,
+              initialDelaySeconds: 0,
+              periodSeconds: 1,
+              successThreshold: 1,
+              timeoutSeconds: 2,
+            },
+            readinessProbe: {
+              httpGet: {
+                path: "/health",
+                port: 869,
+                scheme: "HTTP",
+              },
+              failureThreshold: 1,
+              initialDelaySeconds: 2,
+              periodSeconds: 10,
+              successThreshold: 1,
+              timeoutSeconds: 2,
+            },
+            livenessProbe: {
+              httpGet: {
+                path: "/health",
+                port: 869,
+                scheme: "HTTP",
+              },
+              failureThreshold: 3,
+              initialDelaySeconds: 2,
+              periodSeconds: 10,
+              successThreshold: 1,
+              timeoutSeconds: 2,
+            },
+            volumeMounts: [
+              {
+                name: "riju-bin",
+                mountPath: "/riju-bin",
+                readOnly: true,
+              },
+            ],
+          },
+        ],
+        restartPolicy: "Never",
+      },
+    })
+  ).body;
+  const podIP = await new Promise((resolve, reject) => {
+    setTimeout(() => reject("timed out"), 5 * 60 * 1000);
+    watcher.setCallback(pod.metadata.name, (event, pod) => {
+      if (event == "delete") {
+        reject(new Error("pod was deleted"));
+      } else if (pod.status.phase === "Failed") {
+        reject(new Error("pod status became Failed"));
+      } else if (
+        pod.status.podIP &&
+        lodash.every(pod.status.containerStatuses, (status) => status.ready)
+      ) {
+        resolve(pod.status.podIP);
+      } else {
+        console.log(event, JSON.stringify(pod.status, null, 2));
+      }
+    });
   });
-  console.log(pod);
+  return podIP;
 }
 
 export async function deleteUserSessions(sessionsToDelete) {

backend/sandbox-k8s.js

@@ -26,12 +26,15 @@ async function main() {
   }
   const sessionID = getUUID();
   console.log(`Starting session with UUID ${sessionID}`);
+  const watcher = k8s.watchPods();
   await k8s.createUserSession({
+    watcher,
     sessionID,
     langConfig,
     revisions: {
-      agent: "20221228-023645-invisible-amaranth-sparrow",
+      agent: "20221229-002450-semantic-moccasin-albatross",
       ptyify: "20221228-023645-clean-white-gorilla",
+      langImage: "20221227-195753-forward-harlequin-wolverine",
     },
   });
   // let buffer = "";

env.yaml.bash

@@ -5,6 +5,7 @@ set -euo pipefail
 cd "$(dirname "$0")"
 
 registry_password="$(pwgen -s 20 1)"
+proxy_password="$(pwgen -s 20 1)"
 
 cat <<EOF
 networking:
@@ -25,4 +26,8 @@ registry:
 minio:
   accessKey: "$(head -c16 /dev/urandom | xxd -p)"
   secretKey: "$(head -c16 /dev/urandom | xxd -p)"
+
+proxy:
+  password: "${proxy_password}"
+  htpasswd: "$(htpasswd -nbB admin "${proxy_password}")"
 EOF

k8s/riju-proxy.yaml

@@ -0,0 +1,88 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  namespace: riju
+  name: riju-proxy-config
+data:
+  squid.conf: |
+    cache deny all
+
+    acl riju_src src 10.244.0.0/16
+    http_access deny !riju_src
+
+    acl riju_dst dst 10.244.0.0/16
+    http_access deny !riju_dst
+
+    acl riju_port port 869
+    http_access deny !riju_port
+
+    acl riju_method method GET
+    http_access deny !riju_method
+
+    auth_param basic program /usr/lib/squid/basic_ncsa_auth
+    auth_param basic children 5 startup=5 idle=1
+    auth_param basic realm Riju administrative proxy
+    auth_param basic credentialsttl 24 hours
+
+    acl riju_auth proxy_auth REQUIRED
+    http_access deny !riju_auth
+
+    http_access allow all
+    http_access deny all
+
+    http_port 3128
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  namespace: riju
+  name: riju-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: riju-proxy
+  template:
+    metadata:
+      labels:
+        app: riju-proxy
+    spec:
+      volumes:
+        - name: config
+          configMap:
+            name: riju-proxy-config
+        - name: auth
+          secret:
+            secretName: riju-proxy-auth
+      containers:
+        - name: nginx
+          image: "ubuntu/squid:5.2-22.04_beta"
+          resources: {}
+          readinessProbe:
+            tcpSocket:
+              port: 3128
+            failureThreshold: 1
+            initialDelaySeconds: 2
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 2
+          livenessProbe:
+            tcpSocket:
+              port: 3128
+            failureThreshold: 3
+            initialDelaySeconds: 2
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 2
+          ports:
+            - name: http
+              containerPort: 3128
+          volumeMounts:
+            - name: config
+              mountPath: /etc/squid/squid.conf
+              subPath: squid.conf
+            - name: auth
+              mountPath: /etc/squid/passwd
+              subPath: htpasswd

k8s/secrets.in.yaml

@@ -84,3 +84,12 @@ stringData:
         }
       }
     }
+
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  namespace: riju
+  name: riju-proxy-auth
+data:
+  htpasswd: "{{ .proxy.htpasswd | println | b64enc }}"