v1 for server agent to be hosted on minio
This commit is contained in:
Radon Rosborough
parent
7ea0e4f017
commit
d6134d470c
|
|
@ -0,0 +1,5 @@
|
|||
module github.com/radian-software/riju/agent
|
||||
|
||||
go 1.18
|
||||
|
||||
require github.com/gorilla/websocket v1.5.0 // indirect
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
|
|
@ -0,0 +1,364 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type clientMessage struct {
|
||||
// "stdin"
|
||||
Event string `json:"event"`
|
||||
// contents of stdin
|
||||
Data []byte `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
type serverMessage struct {
|
||||
// "start", "stdout", "stderr", "exit", "warn", "error"
|
||||
Event string `json:"event"`
|
||||
// contents of stdout/stderr
|
||||
Data []byte `json:"data,omitempty"`
|
||||
// error message
|
||||
Text string `json:"text,omitempty"`
|
||||
// exit status
|
||||
ExitStatus *int `json:"exitStatus,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
pingPeriod = 15 * time.Second
|
||||
pongWait = 10 * time.Second
|
||||
writeDeadline = 1 * time.Second
|
||||
)
|
||||
|
||||
var upgrader = websocket.Upgrader{}
|
||||
|
||||
func logWarn(err error) {
|
||||
log.Println(err.Error())
|
||||
}
|
||||
|
||||
func logWarnf(format string, arg ...interface{}) {
|
||||
logWarn(fmt.Errorf(format, arg...))
|
||||
}
|
||||
|
||||
func logError(err error) {
|
||||
log.Println(err.Error())
|
||||
}
|
||||
|
||||
func logErrorf(format string, arg ...interface{}) {
|
||||
logError(fmt.Errorf(format, arg...))
|
||||
}
|
||||
|
||||
func tryClose(obj io.Closer, objName string) {
|
||||
err := obj.Close()
|
||||
if err != nil {
|
||||
logErrorf("error closing %s: %w", objName, err)
|
||||
}
|
||||
}
|
||||
|
||||
func closeWs(ws *websocket.Conn) {
|
||||
err := ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
if err != nil {
|
||||
logErrorf("sending close message: %w", err)
|
||||
}
|
||||
tryClose(ws, "websocket")
|
||||
}
|
||||
|
||||
func send(ws *websocket.Conn, msg *serverMessage) {
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
logErrorf("marshaling message: %w", err)
|
||||
closeWs(ws)
|
||||
return
|
||||
}
|
||||
err = ws.WriteMessage(websocket.TextMessage, data)
|
||||
if err != nil {
|
||||
logErrorf("sending message: %w", err)
|
||||
closeWs(ws)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func fatal(ws *websocket.Conn, err error) {
|
||||
send(ws, &serverMessage{
|
||||
Event: "fatal",
|
||||
Text: err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func fatalf(ws *websocket.Conn, format string, arg ...interface{}) {
|
||||
fatal(ws, fmt.Errorf(format, arg...))
|
||||
}
|
||||
|
||||
func warn(ws *websocket.Conn, err error) {
|
||||
send(ws, &serverMessage{
|
||||
Event: "warn",
|
||||
Text: err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func warnf(ws *websocket.Conn, format string, arg ...interface{}) {
|
||||
warn(ws, fmt.Errorf(format, arg...))
|
||||
}
|
||||
|
||||
func handleClientMessages(ws *websocket.Conn, stdinChan chan<- []byte) {
|
||||
// Close channel after we exit
|
||||
defer close(stdinChan)
|
||||
// Stop processing reads some time after we stop receiving
|
||||
// timely responses to our pings.
|
||||
ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
ws.SetPongHandler(func(string) error {
|
||||
ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
return nil
|
||||
})
|
||||
// Read data and dispatch appropriately. Return on timeout or
|
||||
// error. Caller is responsible for cleanup.
|
||||
for {
|
||||
msgtype, data, err := ws.ReadMessage()
|
||||
if err != nil {
|
||||
fatalf(ws, "reading message: %w", err)
|
||||
return
|
||||
}
|
||||
if msgtype != websocket.TextMessage {
|
||||
fatalf(ws, "received non-text message type %d", msgtype)
|
||||
return
|
||||
}
|
||||
msg := clientMessage{}
|
||||
err = json.Unmarshal(data, &msg)
|
||||
if err != nil {
|
||||
fatalf(ws, "parsing json: %w", err)
|
||||
return
|
||||
}
|
||||
switch msg.Event {
|
||||
case "stdin":
|
||||
stdinChan <- msg.Data
|
||||
default:
|
||||
logWarnf("received unknown event type %s", msg.Event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go
|
||||
func handler(w http.ResponseWriter, r *http.Request) {
|
||||
// Upgrade http connection to websocket
|
||||
ws, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
logErrorf("upgrading connection: %w", err)
|
||||
return
|
||||
}
|
||||
// Close websocket on error or when we exit
|
||||
defer closeWs(ws)
|
||||
// Parse request query parameters; do this after upgrading to
|
||||
// websocket so that we can send errors back on the websocket
|
||||
// which is easier for clients to parse
|
||||
cmdline := r.URL.Query()["cmdline"]
|
||||
if len(cmdline) == 0 {
|
||||
fatalf(ws, "cmdline query parameter missing")
|
||||
return
|
||||
}
|
||||
// Create pipes for communicating with subprocess
|
||||
stdinRead, stdinWrite, err := os.Pipe()
|
||||
if err != nil {
|
||||
fatalf(ws, "creating stdin pipe: %w", err)
|
||||
return
|
||||
}
|
||||
defer tryClose(stdinRead, "read end of stdin pipe")
|
||||
defer tryClose(stdinWrite, "write end of stdin pipe")
|
||||
stdoutRead, stdoutWrite, err := os.Pipe()
|
||||
if err != nil {
|
||||
fatalf(ws, "creating stdout pipe: %w", err)
|
||||
return
|
||||
}
|
||||
defer tryClose(stdoutRead, "read end of stdout pipe")
|
||||
defer tryClose(stdoutWrite, "write end of stdout pipe")
|
||||
stderrRead, stderrWrite, err := os.Pipe()
|
||||
if err != nil {
|
||||
fatalf(ws, "creating stderr pipe: %w", err)
|
||||
return
|
||||
}
|
||||
defer tryClose(stderrRead, "read end of stderr pipe")
|
||||
defer tryClose(stderrWrite, "write end of stderr pipe")
|
||||
// Spawn subprocess
|
||||
proc, err := os.StartProcess(cmdline[0], cmdline, &os.ProcAttr{
|
||||
Files: []*os.File{stdinRead, stdoutWrite, stderrWrite},
|
||||
})
|
||||
if err != nil {
|
||||
fatalf(ws, "spawning process: %w", err)
|
||||
return
|
||||
}
|
||||
// Setup a way for other goroutines to report a fatal error,
|
||||
// use increased capacity to avoid blockage with large number
|
||||
// of write callsites
|
||||
doneChan := make(chan struct{}, 10)
|
||||
// Setup channels and variables to monitor process state
|
||||
waitChan := make(chan struct{}, 1)
|
||||
state := (*os.ProcessState)(nil)
|
||||
// Monitor the process to see when it exits
|
||||
go func() {
|
||||
s, err := proc.Wait()
|
||||
if err != nil {
|
||||
fatalf(ws, "waiting for process to exit: %w", err)
|
||||
} else {
|
||||
state = s
|
||||
}
|
||||
waitChan <- struct{}{}
|
||||
doneChan <- struct{}{}
|
||||
}()
|
||||
// Arrange to send information about the process exit status
|
||||
// if we have obtained it by the time we return
|
||||
defer func() {
|
||||
if state != nil {
|
||||
status := state.ExitCode()
|
||||
send(ws, &serverMessage{
|
||||
Event: "exit",
|
||||
ExitStatus: &status,
|
||||
})
|
||||
}
|
||||
}()
|
||||
// Arrange for subprocess to be killed when we exit
|
||||
defer func() {
|
||||
// See if process has already exited or is about to
|
||||
select {
|
||||
case <-waitChan:
|
||||
return
|
||||
case <-time.NewTimer(500 * time.Millisecond).C:
|
||||
//
|
||||
}
|
||||
// Try killing the process by closing stdin
|
||||
tryClose(stdinWrite, "stdin to child")
|
||||
select {
|
||||
case <-waitChan:
|
||||
return
|
||||
case <-time.NewTimer(500 * time.Millisecond).C:
|
||||
//
|
||||
}
|
||||
// Try killing the process with SIGTERM, SIGINT, then
|
||||
// finally SIGKILL
|
||||
for _, sig := range []os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL} {
|
||||
err = proc.Signal(sig)
|
||||
if err != nil {
|
||||
logWarnf("sending %s to child: %w", sig.String(), err)
|
||||
}
|
||||
select {
|
||||
case <-waitChan:
|
||||
return
|
||||
case <-time.NewTimer(500 * time.Millisecond).C:
|
||||
//
|
||||
}
|
||||
}
|
||||
// We are unable to kill the process
|
||||
fatalf(ws, "unable to kill child")
|
||||
}()
|
||||
// Close our copies of pipe ends passed to subprocess
|
||||
err = stdinRead.Close()
|
||||
if err != nil {
|
||||
fatalf(ws, "closing read end of stdin pipe from parent")
|
||||
return
|
||||
}
|
||||
err = stdoutWrite.Close()
|
||||
if err != nil {
|
||||
fatalf(ws, "closing write end of stdout pipe from parent")
|
||||
return
|
||||
}
|
||||
err = stderrWrite.Close()
|
||||
if err != nil {
|
||||
fatalf(ws, "closing write end of stderr pipe from parent")
|
||||
return
|
||||
}
|
||||
// Handle received messages from client
|
||||
stdinChan := make(chan []byte)
|
||||
go func() {
|
||||
handleClientMessages(ws, stdinChan)
|
||||
doneChan <- struct{}{}
|
||||
}()
|
||||
go func() {
|
||||
for data := range stdinChan {
|
||||
_, err := stdinWrite.Write(data)
|
||||
if err != nil {
|
||||
warnf(ws, "writing to stdin: %w", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
// Send regular pings to ensure we get regular pongs to
|
||||
// satisfy the read deadline on handleClientMessages
|
||||
pingDoneChan := make(chan struct{}, 1)
|
||||
defer func() {
|
||||
pingDoneChan <- struct{}{}
|
||||
}()
|
||||
go func() {
|
||||
ticker := time.NewTicker(pingPeriod)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeDeadline))
|
||||
if err != nil {
|
||||
logErrorf("sending ping: %w", err)
|
||||
doneChan <- struct{}{}
|
||||
return
|
||||
}
|
||||
case <-pingDoneChan:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
// Proxy stdout and stderr back to client
|
||||
go func() {
|
||||
for {
|
||||
buf := make([]byte, 1024)
|
||||
nr, err := stdoutRead.Read(buf)
|
||||
if err != nil {
|
||||
warnf(ws, "reading from stdout: %w", err)
|
||||
return
|
||||
}
|
||||
if nr == 0 {
|
||||
continue
|
||||
}
|
||||
data, err := json.Marshal(&serverMessage{
|
||||
Event: "stdout",
|
||||
Data: buf[:nr],
|
||||
})
|
||||
if err != nil {
|
||||
fatalf(ws, "wrapping stdout in json: %w", err)
|
||||
doneChan <- struct{}{}
|
||||
return
|
||||
}
|
||||
ws.SetWriteDeadline(time.Now().Add(writeDeadline))
|
||||
err = ws.WriteMessage(websocket.TextMessage, data)
|
||||
if err != nil {
|
||||
fatalf(ws, "sending message: %w", err)
|
||||
doneChan <- struct{}{}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
// Wait until either process is exited or a websocket
|
||||
// operation fails
|
||||
<-doneChan
|
||||
// Process and websocket will be cleaned up after return
|
||||
return
|
||||
}
|
||||
|
||||
func main() {
|
||||
port := os.Getenv("RIJU_AGENT_PORT")
|
||||
if port == "" {
|
||||
port = "869"
|
||||
}
|
||||
host := os.Getenv("RIJU_AGENT_HOST")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
fmt.Printf("Listening on http://%s:%s\n", host, port)
|
||||
err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), http.HandlerFunc(handler))
|
||||
if err != nil {
|
||||
logError(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
import * as k8sClient from "@kubernetes/client-node";
|
||||
|
||||
const kubeconfig = new k8sClient.KubeConfig();
|
||||
kubeconfig.loadFromDefault();
|
||||
|
||||
const k8s = kubeconfig.makeApiClient(k8sClient.CoreV1Api);
|
||||
|
||||
async function listUserSessions() {
|
||||
return (await k8s.listNamespacedPod("riju-user")).body.items.map((pod) => ({
|
||||
podName: pod.metadata.name,
|
||||
sessionID: pod.metadata.labels["riju.codes/user-session-id"],
|
||||
}));
|
||||
}
|
||||
|
||||
async function createUserSession({ sessionID, langConfig, revisions }) {
|
||||
await k8s.createNamespacedPod("riju-user", {
|
||||
metadata: {
|
||||
name: `riju-user-session-${sessionID}`,
|
||||
labels: {
|
||||
"riju.codes/user-session-id": sessionID,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
volumes: [
|
||||
{
|
||||
name: "minio-config",
|
||||
secret: {
|
||||
secretName: "minio-user-login",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "riju-bin",
|
||||
emptyDir: {},
|
||||
},
|
||||
],
|
||||
imagePullSecrets: [
|
||||
{
|
||||
name: "registry-user-login",
|
||||
},
|
||||
],
|
||||
initContainers: [
|
||||
{
|
||||
name: "download",
|
||||
image: "minio/mc:RELEASE.2022-12-13T00-23-28Z",
|
||||
resources: {},
|
||||
args: [
|
||||
"sh",
|
||||
"-c",
|
||||
`mc cp riju/agent/${revisions.agent} /riju-bin/agent &&` +
|
||||
`mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify`,
|
||||
],
|
||||
volumeMounts: [
|
||||
{
|
||||
name: "minio-config",
|
||||
mountPath: "/root/.mc",
|
||||
readOnly: true,
|
||||
},
|
||||
{
|
||||
name: "riju-bin",
|
||||
mountPath: "/riju-bin",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
containers: [
|
||||
{
|
||||
name: "session",
|
||||
image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`,
|
||||
resources: {
|
||||
limits: {
|
||||
cpu: "1000m",
|
||||
memory: "4Gi",
|
||||
},
|
||||
},
|
||||
startupProbe: {
|
||||
httpGet: {
|
||||
path: "/health",
|
||||
port: 869,
|
||||
scheme: "HTTP",
|
||||
},
|
||||
failureThreshold: 30,
|
||||
initialDelaySeconds: 0,
|
||||
periodSeconds: 1,
|
||||
successThreshold: 1,
|
||||
timeoutSeconds: 2,
|
||||
},
|
||||
readinessProbe: {
|
||||
httpGet: {
|
||||
path: "/health",
|
||||
port: 869,
|
||||
scheme: "HTTP",
|
||||
},
|
||||
failureThreshold: 1,
|
||||
initialDelaySeconds: 2,
|
||||
periodSeconds: 10,
|
||||
successThreshold: 1,
|
||||
timeoutSeconds: 2,
|
||||
},
|
||||
livenessProbe: {
|
||||
httpGet: {
|
||||
path: "/health",
|
||||
port: 869,
|
||||
scheme: "HTTP",
|
||||
},
|
||||
failureThreshold: 3,
|
||||
initialDelaySeconds: 2,
|
||||
periodSeconds: 10,
|
||||
successThreshold: 1,
|
||||
timeoutSeconds: 2,
|
||||
},
|
||||
volumeMounts: [
|
||||
{
|
||||
name: "riju-bin",
|
||||
mountPath: "/riju-bin",
|
||||
readOnly: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
restartPolicy: "Never",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
---
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
namespace: riju
|
||||
name: minio
|
||||
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
namespace: riju
|
||||
name: minio
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- secrets
|
||||
resourceNames:
|
||||
- minio-keys
|
||||
verbs:
|
||||
- get
|
||||
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
namespace: riju
|
||||
name: minio
|
||||
roleRef:
|
||||
kind: Role
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
name: minio
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: minio
|
||||
|
|
@ -24,7 +24,6 @@ spec:
|
|||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
serviceAccountName: minio
|
||||
containers:
|
||||
- name: minio
|
||||
image: "minio/minio:RELEASE.2022-12-12T19-27-27Z"
|
||||
|
|
|
|||
|
|
@ -63,3 +63,24 @@ metadata:
|
|||
stringData:
|
||||
access-key: "{{ .minio.accessKey }}"
|
||||
secret-key: "{{ .minio.secretKey }}"
|
||||
|
||||
---
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
namespace: riju-user
|
||||
name: minio-user-login
|
||||
stringData:
|
||||
config.json: |
|
||||
{
|
||||
"version": 10,
|
||||
"aliases": {
|
||||
"riju": {
|
||||
"url": "http://minio.riju.svc",
|
||||
"accessKey": "{{ .minio.accessKey }}",
|
||||
"secretKey": "{{ .minio.secretKey }}",
|
||||
"api": "s3v4",
|
||||
"path": "auto"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue