From 48bdc9abf8e331a83ef1d7cad0ec0385b2eff346 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 28 Dec 2022 19:52:59 -0700 Subject: [PATCH] Add/change various things --- agent/go.mod | 5 +- agent/go.sum | 2 + agent/main.go | 28 +++- backend/k8s.js | 282 ++++++++++++++++++++++++++--------------- backend/sandbox-k8s.js | 5 +- env.yaml.bash | 5 + k8s/riju-proxy.yaml | 88 +++++++++++++ k8s/secrets.in.yaml | 9 ++ 8 files changed, 319 insertions(+), 105 deletions(-) create mode 100644 k8s/riju-proxy.yaml diff --git a/agent/go.mod b/agent/go.mod index 8d3a29b..bccdbc0 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -2,4 +2,7 @@ module github.com/radian-software/riju/agent go 1.18 -require github.com/gorilla/websocket v1.5.0 // indirect +require ( + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gorilla/websocket v1.5.0 // indirect +) diff --git a/agent/go.sum b/agent/go.sum index e5a03d4..3b8bc42 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -1,2 +1,4 @@ +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/agent/main.go b/agent/main.go index 3c5b61a..1eda23e 100644 --- a/agent/main.go +++ b/agent/main.go @@ -8,6 +8,7 @@ import ( "os/exec" "time" + "github.com/google/shlex" "github.com/gorilla/websocket" ) @@ -67,6 +68,25 @@ func warnf(ms *ManagedWebsocket, format string, arg ...interface{}) { warn(ms, fmt.Errorf(format, arg...)) } +func getCommandPrefix() []string { + prefix := os.Getenv("RIJU_AGENT_COMMAND_PREFIX") + if prefix == "" { + logErrorf("must specify RIJU_AGENT_COMMAND_PREFIX for security reasons") + os.Exit(1) + } + if prefix == "0" { + return []string{} + } + list, err := shlex.Split(prefix) + if err != nil { + logErrorf("parsing RIJU_AGENT_COMMAND_PREFIX: %w", err) + os.Exit(1) + } + return list +} + +var CommandPrefix = getCommandPrefix() + // https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go func handler(w http.ResponseWriter, r *http.Request) { // Upgrade http connection to websocket @@ -98,6 +118,7 @@ func handler(w http.ResponseWriter, r *http.Request) { fatalf(ms, "cmdline query parameter missing") return } + cmdline = append(CommandPrefix, cmdline...) binary, err := exec.LookPath(cmdline[0]) if err != nil { fatalf(ms, "searching for executable: %w", err) @@ -191,7 +212,12 @@ func main() { host = "0.0.0.0" } fmt.Printf("Listening on http://%s:%s\n", host, port) - err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), http.HandlerFunc(handler)) + mux := http.NewServeMux() + mux.HandleFunc("/exec", handler) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), mux) if err != nil { logError(err) os.Exit(1) diff --git a/backend/k8s.js b/backend/k8s.js index 11d33a0..116e283 100644 --- a/backend/k8s.js +++ b/backend/k8s.js @@ -1,10 +1,54 @@ import * as k8sClient from "@kubernetes/client-node"; +import lodash from "lodash"; const kubeconfig = new k8sClient.KubeConfig(); kubeconfig.loadFromDefault(); const k8s = kubeconfig.makeApiClient(k8sClient.CoreV1Api); +export function watchPods() { + const callbacks = {}; + const pods = {}; + + // https://github.com/kubernetes-client/javascript/blob/1f76ee10c54e33a998abb4686488ccff4285366a/examples/typescript/informer/informer.ts + // + // The watch functionality seems to be wholly undocumented. Copy, + // paste, and pray. + const informer = k8sClient.makeInformer( + kubeconfig, + "/api/v1/namespaces/riju-user/pods", + () => k8s.listNamespacedPod("riju-user") + ); + + for (const event of ["add", "update", "delete"]) { + informer.on(event, (pod) => { + if (pod.metadata.name in callbacks) { + callbacks[pod.metadata.name](event, pod); + } + pods[pod.metadata.name] = pod; + if (event == "delete") { + delete callbacks[pod.metadata.name]; + delete pods[pod.metadata.name]; + } + }); + } + + informer.on("error", (err) => { + console.error(err); + setTimeout(() => informer.start(), 5000); + }); + informer.start(); + + return { + setCallback: (podName, callback) => { + callbacks[podName] = callback; + if (podName in pods) { + callback("add", pods[podName]); + } + }, + }; +} + export async function listUserSessions() { return (await k8s.listNamespacedPod("riju-user")).body.items.map((pod) => ({ podName: pod.metadata.name, @@ -12,116 +56,150 @@ export async function listUserSessions() { })); } -export async function createUserSession({ sessionID, langConfig, revisions }) { - const { body: pod } = await k8s.createNamespacedPod("riju-user", { - metadata: { - name: `riju-user-session-${sessionID}`, - labels: { - "riju.codes/user-session-id": sessionID, +export async function createUserSession({ + watcher, + sessionID, + langConfig, + revisions, +}) { + const pod = ( + await k8s.createNamespacedPod("riju-user", { + metadata: { + name: `riju-user-session-${sessionID}`, + labels: { + "riju.codes/user-session-id": sessionID, + }, }, - }, - spec: { - volumes: [ - { - name: "minio-config", - secret: { - secretName: "minio-user-login", - }, - }, - { - name: "riju-bin", - emptyDir: {}, - }, - ], - imagePullSecrets: [ - { - name: "registry-user-login", - }, - ], - initContainers: [ - { - name: "download", - image: "minio/mc:RELEASE.2022-12-13T00-23-28Z", - resources: {}, - command: ["sh", "-c"], - args: [ - `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` + - `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` + - `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`, - ], - volumeMounts: [ - { - name: "minio-config", - mountPath: "/mc", - readOnly: true, - }, - { - name: "riju-bin", - mountPath: "/riju-bin", - }, - ], - }, - ], - containers: [ - { - name: "session", - image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`, - resources: { - requests: {}, - limits: { - cpu: "1000m", - memory: "4Gi", + spec: { + volumes: [ + { + name: "minio-config", + secret: { + secretName: "minio-user-login", }, }, - startupProbe: { - httpGet: { - path: "/health", - port: 869, - scheme: "HTTP", - }, - failureThreshold: 30, - initialDelaySeconds: 0, - periodSeconds: 1, - successThreshold: 1, - timeoutSeconds: 2, + { + name: "riju-bin", + emptyDir: {}, }, - readinessProbe: { - httpGet: { - path: "/health", - port: 869, - scheme: "HTTP", - }, - failureThreshold: 1, - initialDelaySeconds: 2, - periodSeconds: 10, - successThreshold: 1, - timeoutSeconds: 2, + ], + imagePullSecrets: [ + { + name: "registry-user-login", }, - livenessProbe: { - httpGet: { - path: "/health", - port: 869, - scheme: "HTTP", - }, - failureThreshold: 3, - initialDelaySeconds: 2, - periodSeconds: 10, - successThreshold: 1, - timeoutSeconds: 2, + ], + initContainers: [ + { + name: "download", + image: "minio/mc:RELEASE.2022-12-13T00-23-28Z", + resources: {}, + command: ["sh", "-c"], + args: [ + `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` + + `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` + + `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`, + ], + volumeMounts: [ + { + name: "minio-config", + mountPath: "/mc", + readOnly: true, + }, + { + name: "riju-bin", + mountPath: "/riju-bin", + }, + ], }, - volumeMounts: [ - { - name: "riju-bin", - mountPath: "/riju-bin", - readOnly: true, + ], + containers: [ + { + name: "session", + image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`, + resources: { + requests: {}, + limits: { + cpu: "1000m", + memory: "4Gi", + }, }, - ], - }, - ], - restartPolicy: "Never", - }, + command: ["/riju-bin/agent"], + env: [ + { + name: "RIJU_AGENT_COMMAND_PREFIX", + value: "runuser -u riju --", + }, + ], + securityContext: { + runAsUser: 0, + }, + startupProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 30, + initialDelaySeconds: 0, + periodSeconds: 1, + successThreshold: 1, + timeoutSeconds: 2, + }, + readinessProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 1, + initialDelaySeconds: 2, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 2, + }, + livenessProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 3, + initialDelaySeconds: 2, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 2, + }, + volumeMounts: [ + { + name: "riju-bin", + mountPath: "/riju-bin", + readOnly: true, + }, + ], + }, + ], + restartPolicy: "Never", + }, + }) + ).body; + const podIP = await new Promise((resolve, reject) => { + setTimeout(() => reject("timed out"), 5 * 60 * 1000); + watcher.setCallback(pod.metadata.name, (event, pod) => { + if (event == "delete") { + reject(new Error("pod was deleted")); + } else if (pod.status.phase === "Failed") { + reject(new Error("pod status became Failed")); + } else if ( + pod.status.podIP && + lodash.every(pod.status.containerStatuses, (status) => status.ready) + ) { + resolve(pod.status.podIP); + } else { + console.log(event, JSON.stringify(pod.status, null, 2)); + } + }); }); - console.log(pod); + return podIP; } export async function deleteUserSessions(sessionsToDelete) { diff --git a/backend/sandbox-k8s.js b/backend/sandbox-k8s.js index db616a5..def9a81 100644 --- a/backend/sandbox-k8s.js +++ b/backend/sandbox-k8s.js @@ -26,12 +26,15 @@ async function main() { } const sessionID = getUUID(); console.log(`Starting session with UUID ${sessionID}`); + const watcher = k8s.watchPods(); await k8s.createUserSession({ + watcher, sessionID, langConfig, revisions: { - agent: "20221228-023645-invisible-amaranth-sparrow", + agent: "20221229-002450-semantic-moccasin-albatross", ptyify: "20221228-023645-clean-white-gorilla", + langImage: "20221227-195753-forward-harlequin-wolverine", }, }); // let buffer = ""; diff --git a/env.yaml.bash b/env.yaml.bash index ac1a23e..ebd6c24 100755 --- a/env.yaml.bash +++ b/env.yaml.bash @@ -5,6 +5,7 @@ set -euo pipefail cd "$(dirname "$0")" registry_password="$(pwgen -s 20 1)" +proxy_password="$(pwgen -s 20 1)" cat <