From 29c9cc375d24e9b0e624e740afdfd44b71030c09 Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 16:28:14 -0400 Subject: [PATCH 001/123] Create vlang.yaml --- langs/vlang.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 langs/vlang.yaml diff --git a/langs/vlang.yaml b/langs/vlang.yaml new file mode 100644 index 0000000..592b2ed --- /dev/null +++ b/langs/vlang.yaml @@ -0,0 +1,28 @@ +id: "v" +aliases: + - "vlang" +name: "V" + +install: + manual: | + git clone https://github.com/vlang/v + pushd v + make + popd + +main: "main.v" +template: | + fn main() { + println('hello world') + } + +compile: | + v run main.v + +format: + run: | + v fmt main.v + input: | + fn main() { + println('hello world') + } From d70f17a0880376317b7ca7e174245a133b8c5c0e Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 16:34:31 -0400 Subject: [PATCH 002/123] Add write param to fmt --- langs/vlang.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/vlang.yaml b/langs/vlang.yaml index 592b2ed..cd661bf 100644 --- a/langs/vlang.yaml +++ b/langs/vlang.yaml @@ -21,7 +21,7 @@ compile: | format: run: | - v fmt main.v + v fmt -w main.v input: | fn main() { println('hello world') From 90dc3ff518c68b50ffbf8f2e2ee9029042d26187 Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 16:37:24 -0400 Subject: [PATCH 003/123] Adjust format input --- langs/vlang.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/langs/vlang.yaml b/langs/vlang.yaml index cd661bf..fb4d9ea 100644 --- a/langs/vlang.yaml +++ b/langs/vlang.yaml @@ -13,7 +13,7 @@ install: main: "main.v" template: | fn main() { - println('hello world') + println('Hello, world!') } compile: | @@ -23,6 +23,7 @@ format: run: | v fmt -w main.v input: | - fn main() { - println('hello world') + fn main() + { + println("Hello, world!") } From efde8a36fb32968eaf0eb818a255332d002eb485 Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 16:38:05 -0400 Subject: [PATCH 004/123] Fix tabs --- langs/vlang.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langs/vlang.yaml b/langs/vlang.yaml index fb4d9ea..0d0d00f 100644 --- a/langs/vlang.yaml +++ b/langs/vlang.yaml @@ -13,7 +13,7 @@ install: main: "main.v" template: | fn main() { - println('Hello, world!') + println('Hello, world!') } compile: | @@ -25,5 +25,5 @@ format: input: | fn main() { - println("Hello, world!") + println('Hello, world!') } From f1b4d588f4457c1e7679f00fe25b62dd54b5a02c Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 16:38:37 -0400 Subject: [PATCH 005/123] "Fix" format input quotes --- langs/vlang.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/vlang.yaml b/langs/vlang.yaml index 0d0d00f..10c7e38 100644 --- a/langs/vlang.yaml +++ b/langs/vlang.yaml @@ -25,5 +25,5 @@ format: input: | fn main() { - println('Hello, world!') + println("Hello, world!") } From b3032fad763adc63c0178b5c2ac26d228d9b61dc Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 20:39:42 -0400 Subject: [PATCH 006/123] Rename to v.yaml. --- langs/{vlang.yaml => v.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename langs/{vlang.yaml => v.yaml} (100%) diff --git a/langs/vlang.yaml b/langs/v.yaml similarity index 100% rename from langs/vlang.yaml rename to langs/v.yaml From a9787d084242c1a35eebaf7f7fbfcc37ac80a5d0 Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 20:44:44 -0400 Subject: [PATCH 007/123] Copy exec to /usr/local/bin --- langs/v.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langs/v.yaml b/langs/v.yaml index 10c7e38..ab17948 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -5,9 +5,10 @@ name: "V" install: manual: | - git clone https://github.com/vlang/v + git clone https://github.com/vlang/v.git pushd v make + cp v ${pkg}/usr/local/bin popd main: "main.v" From 7599ab2363609e4c341915e1e69726ee8152ac7e Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 20:45:07 -0400 Subject: [PATCH 008/123] compile: to run: --- langs/v.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/v.yaml b/langs/v.yaml index ab17948..4bcf4f1 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -17,7 +17,7 @@ template: | println('Hello, world!') } -compile: | +run: | v run main.v format: From c11fb18537664b65765a121506939a999aa8ad14 Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 20:45:21 -0400 Subject: [PATCH 009/123] format to stdout. --- langs/v.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/v.yaml b/langs/v.yaml index 4bcf4f1..da36739 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -22,7 +22,7 @@ run: | format: run: | - v fmt -w main.v + v fmt main.v input: | fn main() { From 622ee9d80dfa8dfa88c16e9dccc9350b1c53ed4f Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 21:02:35 -0400 Subject: [PATCH 010/123] Metadata. --- langs/v.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/langs/v.yaml b/langs/v.yaml index da36739..e514b69 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -3,6 +3,26 @@ aliases: - "vlang" name: "V" +info: + year: 2019 + desc: "V is a simple, statically-typed compiled programming language designed for building maintainable software." + ext: + - v + web: + home: "https://vlang.io/" + source: "https://github.com/vlang/v" + wiki: "https://github.com/vlang/v/wiki" + category: general + mode: compiled + syntax: + - c + typing: + - static + paradigm: + - functional + - imperative + usage: personal + install: manual: | git clone https://github.com/vlang/v.git From 393e5bfcb961892e7c75771068811d15ad27fe88 Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Wed, 3 Nov 2021 22:14:31 -0400 Subject: [PATCH 011/123] Fix v.yaml spec issues. --- langs/v.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langs/v.yaml b/langs/v.yaml index e514b69..7b7b22f 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -5,15 +5,15 @@ name: "V" info: year: 2019 - desc: "V is a simple, statically-typed compiled programming language designed for building maintainable software." + desc: "V is a simple, statically-typed compiled programming language designed for building maintainable software" ext: - v web: home: "https://vlang.io/" source: "https://github.com/vlang/v" - wiki: "https://github.com/vlang/v/wiki" category: general mode: compiled + platform: clr syntax: - c typing: From ea5ec85de0a64036e8460560b6a22f9998f629be Mon Sep 17 00:00:00 2001 From: Chris Hallberg Date: Fri, 5 Nov 2021 14:43:00 -0400 Subject: [PATCH 012/123] Remove v alias from verilog. --- langs/verilog.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/langs/verilog.yaml b/langs/verilog.yaml index cfd6eb7..e5f283f 100644 --- a/langs/verilog.yaml +++ b/langs/verilog.yaml @@ -2,7 +2,6 @@ id: "verilog" aliases: - "systemverilog" - "iverilog" - - "v" name: "Verilog" install: From 9103217c4a317e0976ca1efcaf45c421376d9ba9 Mon Sep 17 00:00:00 2001 From: Jason Steving Date: Tue, 9 Nov 2021 11:45:32 -0800 Subject: [PATCH 013/123] Initial pass at onboarding Claro to Riju. --- langs/claro.yaml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 langs/claro.yaml diff --git a/langs/claro.yaml b/langs/claro.yaml new file mode 100644 index 0000000..a74c5fe --- /dev/null +++ b/langs/claro.yaml @@ -0,0 +1,44 @@ +id: "claro" +name: "Claro" + +install: + aptKey: + - "https://bazel.build/bazel-release.pub.gpg" + aptRepo: + - "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" + apt: + - bazel + - default-jdk + manual: | + # Install Claro repo. + install -d "${pkg}/usr/local/bin" + touch "${pkg}/usr/local/bin/MYTEST" + wget "https://github.com/JasonSteving99/claro-lang/archive/refs/tags/v0.1.0.tar.gz" + tar -xf v0.1.0.tar.gz + pushd claro-lang-* + cp -R * "${pkg}/usr/local/bin" + popd + + +setup: | + echo "setting up!" + cp -R /usr/local/bin/* "${HOME}/src" + +main: "src/java/com/claro/claro_programs/conditions.claro" +template: | + # Fill this in later + print("Hello, Riju world!"); + +run: | + echo "Hello, world!" + echo $(bazel --version) + pwd + ls + # This --nojava_header_compilation flag is necessary because Lombok + # fails with error "Turbine is not currently supported by lombok." + # Followed the resolution here: https://githubmemory.com/repo/rzwitserloot/lombok/issues/2911 + # I don't know what any of this means.... but hey it works now *shrug* + bazel run --nojava_header_compilation //src/java/com/claro/claro_programs:conditions_compiled_claro_image + +skip: + - run From fd399a2ad6200706da895be2e3e78f50301a98a2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 15 Nov 2021 17:38:08 -0800 Subject: [PATCH 014/123] Temporarily pin to Ubuntu 21.04 Upgrading is hard and the build system makes it super hard to upgrade languages incrementally right now. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 819a0b4..c2d94d7 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,8 @@ ifeq ($(I),lang) @: $${L} node tools/build-lang-image.js --lang $(L) else ifeq ($(I),ubuntu) - docker pull ubuntu:rolling - hash="$$(docker inspect ubuntu:rolling -f '{{ .Id }}' | sha1sum | awk '{ print $$1 }')"; echo "FROM ubuntu:rolling" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) - + docker pull ubuntu:21.04 + hash="$$(docker inspect ubuntu:21.04 -f '{{ .Id }}' | sha1sum | awk '{ print $$1 }')"; echo "FROM ubuntu:21.04" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) - else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else From 858f612d384c8ac53682076af8dca485c603ce78 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 15 Nov 2021 17:45:25 -0800 Subject: [PATCH 015/123] Pin more --- docker/admin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/admin/Dockerfile b/docker/admin/Dockerfile index ea76193..1981d6d 100644 --- a/docker/admin/Dockerfile +++ b/docker/admin/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:rolling +FROM ubuntu:21.04 COPY docker/admin/install.bash /tmp/ RUN /tmp/install.bash From 066c543e3035caf9356cfc0d9c35c40b70a1cee6 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 15 Nov 2021 18:22:48 -0800 Subject: [PATCH 016/123] Start trying to fix Ante --- langs/ante.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/langs/ante.yaml b/langs/ante.yaml index 2f34309..344e3e1 100644 --- a/langs/ante.yaml +++ b/langs/ante.yaml @@ -9,6 +9,7 @@ install: - cargo - cmake - libssl-dev + - libunwind-dev - pkg-config - python3-distutils manual: | @@ -17,8 +18,8 @@ install: llvmenv init # If compiler is not explicitly set to LLVM, then we get an # error: unrecognized command-line option ‘-Wnewline-eof’. - CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 10.0.0 - llvmenv global 10.0.0 + CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 12.0.1 + llvmenv global 12.0.1 manual: | git clone https://github.com/jfecher/ante.git pushd ante From cf47e68727682e2519b407268ea390e2dade9489 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 15 Nov 2021 18:22:56 -0800 Subject: [PATCH 017/123] [#139] Downgrade to Node 16.x LTS --- docker/admin/install.bash | 2 +- docker/runtime/install.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 962ae65..d64abb5 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -19,7 +19,7 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" -node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" +node_repo="$(curl -sS https://deb.nodesource.com/setup_16.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" tee -a /etc/apt/sources.list.d/custom.list >/dev/null </dev/null < Date: Fri, 3 Dec 2021 04:15:15 -0800 Subject: [PATCH 018/123] ClaroSupport working! This is a massive effort to hack together the Bazel generated jar files and runscript into something that can be deployed on Riju! This all still needs to be cleaned up, but I'm definitely not going to risk losing all this work so it's getting pushed up as is in the meantime. --- langs/claro.yaml | 87 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 73 insertions(+), 14 deletions(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index a74c5fe..8a68013 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -13,32 +13,91 @@ install: # Install Claro repo. install -d "${pkg}/usr/local/bin" touch "${pkg}/usr/local/bin/MYTEST" - wget "https://github.com/JasonSteving99/claro-lang/archive/refs/tags/v0.1.0.tar.gz" - tar -xf v0.1.0.tar.gz - pushd claro-lang-* - cp -R * "${pkg}/usr/local/bin" - popd + # Temporarily try to pull the tarball of the built Bazel repo instead of just the sources. + # wget "https://github.com/JasonSteving99/claro-lang/archive/refs/tags/v0.1.0.tar.gz" + # tar -xf v0.1.0.tar.gz + # pushd claro-lang-* + + # wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro-lang.tar.gz" + # tar -xzvf claro-lang.tar.gz + wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro-lang-bazel-bin.tar.gz" + tar -xzvf claro-lang-bazel-bin.tar.gz + + cp -R external "${pkg}/usr/local/bin" + cp -R src "${pkg}/usr/local/bin" + + # TODO(jsteving) I SHOULD NOW FOLLOW THIS GENERAL APPROACH TO CREATE A RUNSCRIPT: + # Create the shortest classpath we can, by making it relative if possible. + # RUNPATH="${JAVA_RUNFILES}/claro-lang/" + # RUNPATH="${RUNPATH#$PWD/}" + # CLASSPATH="${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/guava/guava/28.1-jre/guava-28.1-jre.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/org/codehaus/mojo/animal-sniffer-annotations/1.18/animal-sniffer-annotations-1.18.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/org/checkerframework/checker-qual/2.8.1/checker-qual-2.8.1.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/errorprone/error_prone_annotations/2.3.2/error_prone_annotations-2.3.2.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/builtins_impls/collections/libcollections_impls.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/builtins_impls/libbuiltins_impls.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libtype.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libbase_type.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/libclaro_type_implementation.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/user_defined_impls/libuser_defined_impls.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libclaro_type_exception.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libconcrete_type.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libtypes.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libparameterized_type.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libtype_provider.jar:${RUNPATH}src/java/com/claro/compiler_backends/interpreted/libscoped_heap.jar:${RUNPATH}src/java/com/claro/libclaro_parser_exception.jar:${RUNPATH}src/java/com/claro/intermediate_representation/expressions/libexpr.jar:${RUNPATH}src/java/com/claro/intermediate_representation/libnode.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libconcrete_types.jar:${RUNPATH}src/java/com/claro/runtime_utilities/libruntime_utilities.jar" + # CLASSPATH+=":${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/org/projectlombok/lombok/1.18.20/lombok-1.18.20.jar" + # CLASSPATH+=":${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/com/google/auto/value/auto-value/1.5.3/auto-value-1.5.3.jar" + # echo $CLASSPATH + # javac -classpath $CLASSPATH Conditions.java + # mkdir com + # mkdir com/claro + # mv Conditions.class com/claro + # jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions.class + # exec $JAVABIN -classpath $CLASSPATH "${ARGS[@]}" + + # popd + wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro_compiler_binary_deploy.jar" + cp claro_compiler_binary_deploy.jar "${pkg}/usr/local/bin" setup: | echo "setting up!" cp -R /usr/local/bin/* "${HOME}/src" + # touch src/java/com/claro/claro_programs/Riju.java + # bazel build --nojava_header_compilation //src/java/com/claro/claro_programs:riju -main: "src/java/com/claro/claro_programs/conditions.claro" +main: "src/java/com/claro/claro_programs/Conditions.claro" template: | - # Fill this in later - print("Hello, Riju world!"); + # Please allow a few minutes for your session to be ready to work with, then have fun! + print("Hello, world!"); run: | - echo "Hello, world!" - echo $(bazel --version) - pwd - ls + # Skip rebuilding the entire compiler all over again and instead just use the pre-built Claro compiler jar. + chmod +rw src/java/com/claro/claro_programs/Conditions.java + java -jar claro_compiler_binary_deploy.jar --java_source --silent --classname=Conditions --package=com.claro \ + < src/java/com/claro/claro_programs/Conditions.claro \ + > src/java/com/claro/claro_programs/Conditions.java + + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # We don't want to depend on Bazel at all for rebuilding, it's just one file changing. Rebuild it using + # the packaged jar files and then update the jar holding that recompiled file and run the Bazel gen'd + # run script which points at all the correct jar runfiles. + pushd src/java/com/claro/claro_programs/ > /dev/null + fancydelim=$(printf '\001') # https://stackoverflow.com/a/25821096/10027370 + + MY_SCRIPT=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/org/projectlombok/lombok/1.18.20/lombok-1.18.20.jar"\n' + MY_SCRIPT+=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/com/google/auto/value/auto-value/1.5.3/auto-value-1.5.3.jar"\n' + MY_SCRIPT+=' javac -classpath $CLASSPATH Conditions.java\n' + # There's an assumption that the dir ./com/claro/ was made before this. + MY_SCRIPT+=' mv Conditions*.class com/claro\n' + MY_SCRIPT+=' jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions*.class\n' + MY_SCRIPT+=' java -classpath $CLASSPATH "${ARGS[@]}"\n' + # We insert our own exit here to short-circuit the call to JAVABIN which is invalid on this host machine. + MY_SCRIPT+=' exit $?\n' + + sed -n "H;\${x;s/\n//;s${fancydelim}create_and_run_classpath_jar\nelse\n${fancydelim}&${MY_SCRIPT}\n${fancydelim};p;}" conditions_compiled_claro_image > conditions_compiled_claro_image2 + + chmod 777 conditions_compiled_claro_image + cat conditions_compiled_claro_image2 > conditions_compiled_claro_image + + ./conditions_compiled_claro_image + + popd > /dev/null + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + # This --nojava_header_compilation flag is necessary because Lombok # fails with error "Turbine is not currently supported by lombok." # Followed the resolution here: https://githubmemory.com/repo/rzwitserloot/lombok/issues/2911 # I don't know what any of this means.... but hey it works now *shrug* - bazel run --nojava_header_compilation //src/java/com/claro/claro_programs:conditions_compiled_claro_image + # bazel run --nojava_header_compilation --ui_event_filters=-info,-stdout,-stderr --noshow_progress //src/java/com/claro/claro_programs:riju + # bazel run --nojava_header_compilation --ui_event_filters=-info,-stdout,-stderr --noshow_progress //src/java/com/claro/claro_programs:conditions_compiled_claro_image skip: - - run + - run From c360f89a09f1d95f56539e10dcbea44e2cbd0fdf Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 5 Dec 2021 17:59:01 -0800 Subject: [PATCH 019/123] Add financial data for Nov 2021 --- financials/2021-11/breakdown.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 financials/2021-11/breakdown.txt diff --git a/financials/2021-11/breakdown.txt b/financials/2021-11/breakdown.txt new file mode 100644 index 0000000..3462f7d --- /dev/null +++ b/financials/2021-11/breakdown.txt @@ -0,0 +1,17 @@ +Riju :: $133.50 + EC2 :: $108.22 + Data Transfer :: $0.02 + EBS Snapshot :: $3.79 + EBS Volume :: $27.58 + EBS Volume :: $27.58 + gp2 :: $1.04 + gp3 :: $26.55 + Instance :: $76.82 + t3.large :: $76.82 + ECR :: $5.34 + Storage :: $5.34 + ELB :: $19.83 + Data Transfer :: $0.69 + LCUs :: $0.10 + Load Balancer :: $19.04 + S3 :: $0.11 From 1104bfb3a5dbaabdcb437db24f7cc762ae0d10bc Mon Sep 17 00:00:00 2001 From: Jason Steving Date: Mon, 6 Dec 2021 01:54:33 -0800 Subject: [PATCH 020/123] ClaroSupport working efficiently! And NOW.....A REPL on RIJU! This is the final push towards implementing legitimate support for Claro in Riju! I've now figured out how to package up all of the Bazel generated build resources (jars) and the Bazel generated run script and have pieced everything together so that rebuilds and initial page loads are all as efficient as I can make them so far accounting for the intermediate step of compiling to Java. The general process is to package everything needed from Bazel in a tarball and then to dynamically insert a custom script to the Bazel generated run script in bazel-bin to rebuild and update the jar file for the latest modification to the file. This allows me to canibalize Bazel's logic for determining where all the proper "runfiles" a.k.a. jars are located in that tarball. With this approach I'm also able to trivially support the Claro REPL in Riju as well!!!!! This is the greatest unexpected surprise :D. --- langs/claro.yaml | 126 +++++++++++++++++++---------------------------- 1 file changed, 52 insertions(+), 74 deletions(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index 8a68013..c611fe2 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -2,102 +2,80 @@ id: "claro" name: "Claro" install: - aptKey: - - "https://bazel.build/bazel-release.pub.gpg" - aptRepo: - - "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" apt: - - bazel - default-jdk manual: | - # Install Claro repo. install -d "${pkg}/usr/local/bin" - touch "${pkg}/usr/local/bin/MYTEST" - # Temporarily try to pull the tarball of the built Bazel repo instead of just the sources. - # wget "https://github.com/JasonSteving99/claro-lang/archive/refs/tags/v0.1.0.tar.gz" - # tar -xf v0.1.0.tar.gz - # pushd claro-lang-* - # wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro-lang.tar.gz" - # tar -xzvf claro-lang.tar.gz - wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro-lang-bazel-bin.tar.gz" + # Pull resources from the latest Claro repo release. + ver="$(latest_release JasonSteving99/claro-lang)" + + # Pull the tarball of the built Bazel repo's bazel-bin instead of just the sources. + wget "https://github.com/JasonSteving99/claro-lang/releases/download/${ver}/claro-lang-bazel-bin.tar.gz" tar -xzvf claro-lang-bazel-bin.tar.gz - cp -R external "${pkg}/usr/local/bin" - cp -R src "${pkg}/usr/local/bin" + ##################################################################################################### + # We don't want to depend on Bazel at all for rebuilding, it's just one file changing. Rebuild it using + # the packaged jar files and then update the jar holding that recompiled file and run the Bazel gen'd + # run script which points at all the correct jar runfiles. First though, we need to slightly modify + # the Bazel gen'd runscript to rebuild using the packaged jars for us (without explicitly rerunning Bazel + # itself since this is super slow and involves starting up a new Bazel server...). + pushd claro_programs/ > /dev/null + fancydelim=$(printf '\001') # https://stackoverflow.com/a/25821096/10027370 - # TODO(jsteving) I SHOULD NOW FOLLOW THIS GENERAL APPROACH TO CREATE A RUNSCRIPT: - # Create the shortest classpath we can, by making it relative if possible. - # RUNPATH="${JAVA_RUNFILES}/claro-lang/" - # RUNPATH="${RUNPATH#$PWD/}" - # CLASSPATH="${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/guava/guava/28.1-jre/guava-28.1-jre.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/org/codehaus/mojo/animal-sniffer-annotations/1.18/animal-sniffer-annotations-1.18.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/org/checkerframework/checker-qual/2.8.1/checker-qual-2.8.1.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar:${RUNPATH}../maven/v1/https/jcenter.bintray.com/com/google/errorprone/error_prone_annotations/2.3.2/error_prone_annotations-2.3.2.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/builtins_impls/collections/libcollections_impls.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/builtins_impls/libbuiltins_impls.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libtype.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libbase_type.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/libclaro_type_implementation.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/impls/user_defined_impls/libuser_defined_impls.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libclaro_type_exception.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libconcrete_type.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libtypes.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libparameterized_type.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libtype_provider.jar:${RUNPATH}src/java/com/claro/compiler_backends/interpreted/libscoped_heap.jar:${RUNPATH}src/java/com/claro/libclaro_parser_exception.jar:${RUNPATH}src/java/com/claro/intermediate_representation/expressions/libexpr.jar:${RUNPATH}src/java/com/claro/intermediate_representation/libnode.jar:${RUNPATH}src/java/com/claro/intermediate_representation/types/libconcrete_types.jar:${RUNPATH}src/java/com/claro/runtime_utilities/libruntime_utilities.jar" - # CLASSPATH+=":${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/org/projectlombok/lombok/1.18.20/lombok-1.18.20.jar" - # CLASSPATH+=":${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/com/google/auto/value/auto-value/1.5.3/auto-value-1.5.3.jar" - # echo $CLASSPATH - # javac -classpath $CLASSPATH Conditions.java - # mkdir com - # mkdir com/claro - # mv Conditions.class com/claro - # jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions.class - # exec $JAVABIN -classpath $CLASSPATH "${ARGS[@]}" + # MY_SCRIPT=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/org/projectlombok/lombok/1.18.20/lombok-1.18.20.jar"\n' + # MY_SCRIPT+=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/com/google/auto/value/auto-value/1.5.3/auto-value-1.5.3.jar"\n' + # For the purpose of rebuilding, we need lombok and autovalue on the classpath. + MY_SCRIPT=' REBUILD_CLASSPATH="${CLASSPATH}"\n' + MY_SCRIPT+=' REBUILD_CLASSPATH+=":lombok-1.18.20.jar"\n' + MY_SCRIPT+=' REBUILD_CLASSPATH+=":auto-value-1.5.3.jar"\n' + MY_SCRIPT+=' javac -classpath $REBUILD_CLASSPATH Conditions.java\n' + # There's an assumption that the dir ./com/claro/ was made in the tarball before this. + MY_SCRIPT+=' mv Conditions*.class com/claro\n' + MY_SCRIPT+=' jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions*.class\n' + MY_SCRIPT+=' java -classpath $CLASSPATH "${ARGS[@]}"\n' + # We insert our own exit here to short-circuit the Bazel run script's call to JAVABIN which is + # invalid on this host machine. + MY_SCRIPT+=' exit $?\n' + + # Insert MY_SCRIPT into the Bazel run script just before java gets executed. We're reusing Bazel's run + # script basically just to get a conveniently curated CLASSPATH variable generated to point to all the + # randomly scattered jar files that Bazel places throughout bazel-bin/. + sed -n "H;\${x;s/\n//;s${fancydelim}create_and_run_classpath_jar\nelse\n${fancydelim}&${MY_SCRIPT}\n${fancydelim};p;}" conditions_compiled_claro_image > conditions_compiled_claro_image2 + + # I'm not good enough with sed to figure out how to avoid this extra temporary stop in a second file. Oh well. + chmod 777 conditions_compiled_claro_image + cat conditions_compiled_claro_image2 > conditions_compiled_claro_image + + popd > /dev/null + ##################################################################################################### + + cp -R claro_programs "${pkg}/usr/local/bin" - # popd wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro_compiler_binary_deploy.jar" cp claro_compiler_binary_deploy.jar "${pkg}/usr/local/bin" - setup: | - echo "setting up!" cp -R /usr/local/bin/* "${HOME}/src" - # touch src/java/com/claro/claro_programs/Riju.java - # bazel build --nojava_header_compilation //src/java/com/claro/claro_programs:riju -main: "src/java/com/claro/claro_programs/Conditions.claro" +main: "claro_programs/Conditions.claro" template: | - # Please allow a few minutes for your session to be ready to work with, then have fun! + # Thanks for trying out Claro during its early development stages! + # To learn Claro by example, check out: + # https://clarolang.com/tree/main/src/java/com/claro/claro_programs + print("Hello, world!"); run: | # Skip rebuilding the entire compiler all over again and instead just use the pre-built Claro compiler jar. - chmod +rw src/java/com/claro/claro_programs/Conditions.java + chmod +rw claro_programs/Conditions.java java -jar claro_compiler_binary_deploy.jar --java_source --silent --classname=Conditions --package=com.claro \ - < src/java/com/claro/claro_programs/Conditions.claro \ - > src/java/com/claro/claro_programs/Conditions.java - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # We don't want to depend on Bazel at all for rebuilding, it's just one file changing. Rebuild it using - # the packaged jar files and then update the jar holding that recompiled file and run the Bazel gen'd - # run script which points at all the correct jar runfiles. - pushd src/java/com/claro/claro_programs/ > /dev/null - fancydelim=$(printf '\001') # https://stackoverflow.com/a/25821096/10027370 - - MY_SCRIPT=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/org/projectlombok/lombok/1.18.20/lombok-1.18.20.jar"\n' - MY_SCRIPT+=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/com/google/auto/value/auto-value/1.5.3/auto-value-1.5.3.jar"\n' - MY_SCRIPT+=' javac -classpath $CLASSPATH Conditions.java\n' - # There's an assumption that the dir ./com/claro/ was made before this. - MY_SCRIPT+=' mv Conditions*.class com/claro\n' - MY_SCRIPT+=' jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions*.class\n' - MY_SCRIPT+=' java -classpath $CLASSPATH "${ARGS[@]}"\n' - # We insert our own exit here to short-circuit the call to JAVABIN which is invalid on this host machine. - MY_SCRIPT+=' exit $?\n' - - sed -n "H;\${x;s/\n//;s${fancydelim}create_and_run_classpath_jar\nelse\n${fancydelim}&${MY_SCRIPT}\n${fancydelim};p;}" conditions_compiled_claro_image > conditions_compiled_claro_image2 - - chmod 777 conditions_compiled_claro_image - cat conditions_compiled_claro_image2 > conditions_compiled_claro_image + < claro_programs/Conditions.claro \ + > claro_programs/Conditions.java + pushd claro_programs/ > /dev/null ./conditions_compiled_claro_image - popd > /dev/null - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - # This --nojava_header_compilation flag is necessary because Lombok - # fails with error "Turbine is not currently supported by lombok." - # Followed the resolution here: https://githubmemory.com/repo/rzwitserloot/lombok/issues/2911 - # I don't know what any of this means.... but hey it works now *shrug* - # bazel run --nojava_header_compilation --ui_event_filters=-info,-stdout,-stderr --noshow_progress //src/java/com/claro/claro_programs:riju - # bazel run --nojava_header_compilation --ui_event_filters=-info,-stdout,-stderr --noshow_progress //src/java/com/claro/claro_programs:conditions_compiled_claro_image - -skip: - - run +repl: | + java -jar claro_compiler_binary_deploy.jar --repl --silent \ No newline at end of file From 733ccee274d1a292d52edff87fd27d7b8b12b9f9 Mon Sep 17 00:00:00 2001 From: Jason Steving Date: Mon, 6 Dec 2021 01:57:02 -0800 Subject: [PATCH 021/123] ClaroSupport working efficiently! And NOW.....A REPL on RIJU! This is the final push towards implementing legitimate support for Claro in Riju! I've now figured out how to package up all of the Bazel generated build resources (jars) and the Bazel generated run script and have pieced everything together so that rebuilds and initial page loads are all as efficient as I can make them so far accounting for the intermediate step of compiling to Java. The general process is to package everything needed from Bazel in a tarball and then to dynamically insert a custom script to the Bazel generated run script in bazel-bin to rebuild and update the jar file for the latest modification to the file. This allows me to canibalize Bazel's logic for determining where all the proper "runfiles" a.k.a. jars are located in that tarball. With this approach I'm also able to trivially support the Claro REPL in Riju as well!!!!! This is the greatest unexpected surprise :D. --- langs/claro.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index c611fe2..a4fb577 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -23,8 +23,6 @@ install: pushd claro_programs/ > /dev/null fancydelim=$(printf '\001') # https://stackoverflow.com/a/25821096/10027370 - # MY_SCRIPT=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/org/projectlombok/lombok/1.18.20/lombok-1.18.20.jar"\n' - # MY_SCRIPT+=' CLASSPATH+=":\${RUNPATH}../../../../../../../external/maven/v1/https/jcenter.bintray.com/com/google/auto/value/auto-value/1.5.3/auto-value-1.5.3.jar"\n' # For the purpose of rebuilding, we need lombok and autovalue on the classpath. MY_SCRIPT=' REBUILD_CLASSPATH="${CLASSPATH}"\n' MY_SCRIPT+=' REBUILD_CLASSPATH+=":lombok-1.18.20.jar"\n' From 5e4d0bb18fe91f98970aa529c8e8b3f02960a0da Mon Sep 17 00:00:00 2001 From: Jason Steving Date: Mon, 6 Dec 2021 02:09:21 -0800 Subject: [PATCH 022/123] ClaroSupport working efficiently! And NOW.....A REPL on RIJU! This is the final push towards implementing legitimate support for Claro in Riju! I've now figured out how to package up all of the Bazel generated build resources (jars) and the Bazel generated run script and have pieced everything together so that rebuilds and initial page loads are all as efficient as I can make them so far accounting for the intermediate step of compiling to Java. The general process is to package everything needed from Bazel in a tarball and then to dynamically insert a custom script to the Bazel generated run script in bazel-bin to rebuild and update the jar file for the latest modification to the file. This allows me to canibalize Bazel's logic for determining where all the proper "runfiles" a.k.a. jars are located in that tarball. With this approach I'm also able to trivially support the Claro REPL in Riju as well!!!!! This is the greatest unexpected surprise :D. --- langs/claro.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index a4fb577..aa4716f 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -44,6 +44,7 @@ install: # I'm not good enough with sed to figure out how to avoid this extra temporary stop in a second file. Oh well. chmod 777 conditions_compiled_claro_image cat conditions_compiled_claro_image2 > conditions_compiled_claro_image + rm conditions_compiled_claro_image2 popd > /dev/null ##################################################################################################### @@ -69,11 +70,13 @@ run: | chmod +rw claro_programs/Conditions.java java -jar claro_compiler_binary_deploy.jar --java_source --silent --classname=Conditions --package=com.claro \ < claro_programs/Conditions.claro \ - > claro_programs/Conditions.java - - pushd claro_programs/ > /dev/null - ./conditions_compiled_claro_image - popd > /dev/null + > claro_programs/Conditions.java \ + && \ + pushd claro_programs/ > /dev/null \ + && \ + ./conditions_compiled_claro_image \ + && \ + popd > /dev/null repl: | java -jar claro_compiler_binary_deploy.jar --repl --silent \ No newline at end of file From a024cdefa17b7ac4f9a603c174563fb4634b5d2e Mon Sep 17 00:00:00 2001 From: Jason Steving Date: Mon, 6 Dec 2021 02:22:25 -0800 Subject: [PATCH 023/123] ClaroSupport working efficiently! And NOW.....A REPL on RIJU! This is the final push towards implementing legitimate support for Claro in Riju! I've now figured out how to package up all of the Bazel generated build resources (jars) and the Bazel generated run script and have pieced everything together so that rebuilds and initial page loads are all as efficient as I can make them so far accounting for the intermediate step of compiling to Java. The general process is to package everything needed from Bazel in a tarball and then to dynamically insert a custom script to the Bazel generated run script in bazel-bin to rebuild and update the jar file for the latest modification to the file. This allows me to canibalize Bazel's logic for determining where all the proper "runfiles" a.k.a. jars are located in that tarball. With this approach I'm also able to trivially support the Claro REPL in Riju as well!!!!! This is the greatest unexpected surprise :D. --- langs/claro.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/langs/claro.yaml b/langs/claro.yaml index aa4716f..4aaeac3 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -1,6 +1,27 @@ id: "claro" name: "Claro" +info: + impl: "Claro" + version: "0.1.0" + year: 2021 + desc: "Optionally compiled or interpreted, high-level, toy programming language, with an eye towards providing standardized Software Engineering best practices out of the box" + ext: + - claro + web: + home: "https://clarolang.com/" + source: "https://github.com/JasonSteving99/claro-lang/" + category: recreational + mode: compiled or interpreted + platform: Bazel + syntax: + - java + typing: static + paradigm: + - functional + - imperative + usage: literally just Jason Steving + install: apt: - default-jdk From b950e503dbd24184c3f6566254eb50e3490e306e Mon Sep 17 00:00:00 2001 From: Jason Steving Date: Mon, 6 Dec 2021 02:32:25 -0800 Subject: [PATCH 024/123] ClaroSupport working efficiently! And NOW.....A REPL on RIJU! This is the final push towards implementing legitimate support for Claro in Riju! I've now figured out how to package up all of the Bazel generated build resources (jars) and the Bazel generated run script and have pieced everything together so that rebuilds and initial page loads are all as efficient as I can make them so far accounting for the intermediate step of compiling to Java. The general process is to package everything needed from Bazel in a tarball and then to dynamically insert a custom script to the Bazel generated run script in bazel-bin to rebuild and update the jar file for the latest modification to the file. This allows me to canibalize Bazel's logic for determining where all the proper "runfiles" a.k.a. jars are located in that tarball. With this approach I'm also able to trivially support the Claro REPL in Riju as well!!!!! This is the greatest unexpected surprise :D. --- langs/claro.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index 4aaeac3..4d9c068 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -100,4 +100,10 @@ run: | popd > /dev/null repl: | - java -jar claro_compiler_binary_deploy.jar --repl --silent \ No newline at end of file + java -jar claro_compiler_binary_deploy.jar --repl --silent + +input: | + print(123 * 234); + +skip: + - runrepl From 2dda21f0af28c153050e48f32884ed3faad8a48d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 11 Dec 2021 11:16:46 -0800 Subject: [PATCH 025/123] Roll back to working version of Ante This reverts commit 066c543e3035caf9356cfc0d9c35c40b70a1cee6. --- docker/ci/Dockerfile | 2 +- langs/ante.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index 8098cb4..e15397b 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:rolling +FROM ubuntu:21.04 COPY docker/ci/install.bash /tmp/ RUN /tmp/install.bash diff --git a/langs/ante.yaml b/langs/ante.yaml index 344e3e1..e061695 100644 --- a/langs/ante.yaml +++ b/langs/ante.yaml @@ -9,7 +9,6 @@ install: - cargo - cmake - libssl-dev - - libunwind-dev - pkg-config - python3-distutils manual: | @@ -18,11 +17,12 @@ install: llvmenv init # If compiler is not explicitly set to LLVM, then we get an # error: unrecognized command-line option ‘-Wnewline-eof’. - CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 12.0.1 - llvmenv global 12.0.1 + CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 10.0.0 + llvmenv global 10.0.0 manual: | - git clone https://github.com/jfecher/ante.git + git clone https://github.com/jfecher/ante.git -n pushd ante + git checkout 6e61e9f7ed9300ed68f6b9459b69f027e09a0c42 LLVM_SYS_100_PREFIX="$(llvmenv prefix)" cargo build --release install -d "${pkg}/opt/ante" install -d "${pkg}/usr/local/bin" From 1d57528328bc173ec94019c66d54386d30f739ae Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 11 Dec 2021 11:48:58 -0800 Subject: [PATCH 026/123] Sigh --- langs/ante.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/langs/ante.yaml b/langs/ante.yaml index e061695..539fd76 100644 --- a/langs/ante.yaml +++ b/langs/ante.yaml @@ -17,12 +17,12 @@ install: llvmenv init # If compiler is not explicitly set to LLVM, then we get an # error: unrecognized command-line option ‘-Wnewline-eof’. - CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 10.0.0 - llvmenv global 10.0.0 + CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 10.0.1 + llvmenv global 10.0.1 manual: | git clone https://github.com/jfecher/ante.git -n pushd ante - git checkout 6e61e9f7ed9300ed68f6b9459b69f027e09a0c42 + git checkout ba940f3b492fb448a6a73b139403eefa7a0daedc LLVM_SYS_100_PREFIX="$(llvmenv prefix)" cargo build --release install -d "${pkg}/opt/ante" install -d "${pkg}/usr/local/bin" From 8e97a846adec516c2a551d1254f140604e68ee48 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 11 Dec 2021 11:53:41 -0800 Subject: [PATCH 027/123] Aaaaaaaaaaaaaaaaa --- langs/ante.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/ante.yaml b/langs/ante.yaml index 539fd76..bd1be00 100644 --- a/langs/ante.yaml +++ b/langs/ante.yaml @@ -16,7 +16,7 @@ install: cargo install llvmenv llvmenv init # If compiler is not explicitly set to LLVM, then we get an - # error: unrecognized command-line option ‘-Wnewline-eof’. + # error: unrecognized command-line option '-Wnewline-eof'. CC=/usr/bin/clang CXX=/usr/bin/clang++ llvmenv build-entry -G Makefile -j$(nproc) 10.0.1 llvmenv global 10.0.1 manual: | From 0eddf34215dac6e3069ab1e3b53fdc40e332c1a7 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 11 Dec 2021 12:13:22 -0800 Subject: [PATCH 028/123] Downgrade --- packer/web.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index ff1125f..c20557a 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -35,7 +35,7 @@ variable "supervisor_access_token" { data "amazon-ami" "ubuntu" { filters = { - name = "ubuntu/images/hvm-ssd/ubuntu-*-21.10-amd64-server-*" + name = "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*" root-device-type = "ebs" virtualization-type = "hvm" } From 61890838cda00d28347ed0c55d71c0f60c8ae494 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 11 Dec 2021 12:27:31 -0800 Subject: [PATCH 029/123] Carp has become too slow to run --- langs/carp.yaml | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 langs/carp.yaml diff --git a/langs/carp.yaml b/langs/carp.yaml deleted file mode 100644 index 4aca952..0000000 --- a/langs/carp.yaml +++ /dev/null @@ -1,49 +0,0 @@ -id: "carp" -name: "Carp" - -info: - year: 2015 - desc: "Programming language designed to work well for interactive and performance sensitive use cases like games, sound synthesis and visualizations" - ext: carp - web: - source: "https://github.com/carp-lang/Carp" - category: general - mode: interpreted - platform: [] - syntax: lisp - typing: static - paradigm: - - functional - - imperative - usage: [] - -install: - manual: | - install -d "${pkg}/opt/carp" - install -d "${pkg}/usr/local/bin" - - ver="$(curl -sSL "https://api.github.com/repos/carp-lang/Carp/releases" | jq '.[].tag_name' -r | grep Linux | sort -rV | grep -Eo '[0-9.]+' | head -n1)" - wget "https://github.com/carp-lang/Carp/releases/download/v${ver}_Linux/v${ver}.zip" -O carp.zip - unzip carp.zip - mv "v${ver}/bin"/* "${pkg}/usr/local/bin/" - mv "v${ver}/core" "${pkg}/opt/carp/" - -repl: | - CARP_DIR=/opt/carp carp -input: | - (* 123 234) - -main: "main.carp" -template: | - (use IO) - - (println "Hello, world!") - -run: | - CARP_DIR=/opt/carp carp main.carp - -scope: - code: | - (def x (* 123 234)) - -timeoutFactor: 8 From 0b7c77ebae6db7aa618b9577f543f3437b887126 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 11 Dec 2021 13:55:20 -0800 Subject: [PATCH 030/123] Lose more sanity --- docker/app/install-build.bash | 2 +- langs/kotlin.yaml | 2 ++ langs/purescript.yaml | 2 +- langs/python.yaml | 3 +++ langs/qsharp.yaml | 2 ++ langs/red.yaml | 2 +- langs/scala.yaml | 5 +++++ langs/unison.yaml | 4 ++-- 8 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docker/app/install-build.bash b/docker/app/install-build.bash index 5b80794..42281fc 100755 --- a/docker/app/install-build.bash +++ b/docker/app/install-build.bash @@ -15,7 +15,7 @@ curl -sSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - ubuntu_ver="$(lsb_release -rs)" ubuntu_name="$(lsb_release -cs)" -node_repo="$(curl -sS https://deb.nodesource.com/setup_current.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" +node_repo="$(curl -sS https://deb.nodesource.com/setup_16.x | grep NODEREPO= | grep -Eo 'node_[0-9]+\.x' | head -n1)" tee -a /etc/apt/sources.list.d/custom.list >/dev/null < .purs-repl; spago repl; else echo 'import Prelude' > .purs-repl; spago repl -d; fi -timeoutFactor: 2 +timeoutFactor: 8 diff --git a/langs/python.yaml b/langs/python.yaml index 6a63fc1..a024b06 100644 --- a/langs/python.yaml +++ b/langs/python.yaml @@ -90,3 +90,6 @@ lsp: InterpreterPath: /usr/bin/python3 code: "import func" item: "functools" + +skip: + - lsp diff --git a/langs/qsharp.yaml b/langs/qsharp.yaml index 98f7a42..19abf41 100644 --- a/langs/qsharp.yaml +++ b/langs/qsharp.yaml @@ -60,3 +60,5 @@ template: | run: | dotnet run --project main + +timeoutFactor: 4 diff --git a/langs/red.yaml b/langs/red.yaml index 8c69509..8bcf716 100644 --- a/langs/red.yaml +++ b/langs/red.yaml @@ -23,7 +23,7 @@ setup: | repl: | "$(which red)" input: | - DELAY: 1 + DELAY: 5 123 * 234 main: "main.red" diff --git a/langs/scala.yaml b/langs/scala.yaml index 08b7f1f..78575c9 100644 --- a/langs/scala.yaml +++ b/langs/scala.yaml @@ -7,6 +7,9 @@ install: repl: | scala +input: | + DELAY: 5 + 123 * 234 main: "main.scala" template: | @@ -18,3 +21,5 @@ run: | scope: code: | val x = 123 * 234 + +timeoutFactor: 2 diff --git a/langs/unison.yaml b/langs/unison.yaml index a48dd64..9e9d9b3 100644 --- a/langs/unison.yaml +++ b/langs/unison.yaml @@ -63,11 +63,11 @@ scope: code: | x = 123 * 234 input: | - DELAY: 5 + DELAY: 15 load main.u DELAY: 5 add x DELAY: 5 display x -timeoutFactor: 2 +timeoutFactor: 4 From 76bc7825af0256859eb6124386fe7d8ea7e15402 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 11:04:42 -0800 Subject: [PATCH 031/123] Various improvements --- langs/claro.yaml | 141 +++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 72 deletions(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index 4d9c068..6a18a54 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -3,107 +3,104 @@ name: "Claro" info: impl: "Claro" - version: "0.1.0" year: 2021 - desc: "Optionally compiled or interpreted, high-level, toy programming language, with an eye towards providing standardized Software Engineering best practices out of the box" + desc: "High-level toy programming language providing standardized Software Engineering best practices out of the box" ext: - claro web: home: "https://clarolang.com/" - source: "https://github.com/JasonSteving99/claro-lang/" - category: recreational - mode: compiled or interpreted - platform: Bazel + source: "https://github.com/JasonSteving99/claro-lang" + category: general + mode: + - compiled + - interpreted + platform: [] syntax: - - java + - c typing: static paradigm: - functional - imperative - usage: literally just Jason Steving + usage: personal install: - apt: - - default-jdk - manual: | - install -d "${pkg}/usr/local/bin" + apt: + - default-jdk + manual: | + install -d "${pkg}/opt/claro/programs" + install -d "${pkg}/usr/local/bin" - # Pull resources from the latest Claro repo release. - ver="$(latest_release JasonSteving99/claro-lang)" + # Pull resources from the latest Claro repo release. + ver="$(latest_release JasonSteving99/claro-lang | sed 's/^v//')" - # Pull the tarball of the built Bazel repo's bazel-bin instead of just the sources. - wget "https://github.com/JasonSteving99/claro-lang/releases/download/${ver}/claro-lang-bazel-bin.tar.gz" - tar -xzvf claro-lang-bazel-bin.tar.gz + # Pull the tarball of the built Bazel repo's bazel-bin instead of just the sources. + wget "https://github.com/JasonSteving99/claro-lang/releases/download/v${ver}/claro-lang-bazel-bin.tar.gz" + tar -xf claro-lang-bazel-bin.tar.gz - ##################################################################################################### - # We don't want to depend on Bazel at all for rebuilding, it's just one file changing. Rebuild it using - # the packaged jar files and then update the jar holding that recompiled file and run the Bazel gen'd - # run script which points at all the correct jar runfiles. First though, we need to slightly modify - # the Bazel gen'd runscript to rebuild using the packaged jars for us (without explicitly rerunning Bazel - # itself since this is super slow and involves starting up a new Bazel server...). - pushd claro_programs/ > /dev/null - fancydelim=$(printf '\001') # https://stackoverflow.com/a/25821096/10027370 + ##################################################################################################### + # We don't want to depend on Bazel at all for rebuilding, it's just one file changing. Rebuild it using + # the packaged jar files and then update the jar holding that recompiled file and run the Bazel gen'd + # run script which points at all the correct jar runfiles. First though, we need to slightly modify + # the Bazel gen'd runscript to rebuild using the packaged jars for us (without explicitly rerunning Bazel + # itself since this is super slow and involves starting up a new Bazel server...). + pushd claro_programs + read -r -d '' MY_SCRIPT <<"EOF" ||: + REBUILD_CLASSPATH="${CLASSPATH}" # For the purpose of rebuilding, we need lombok and autovalue on the classpath. - MY_SCRIPT=' REBUILD_CLASSPATH="${CLASSPATH}"\n' - MY_SCRIPT+=' REBUILD_CLASSPATH+=":lombok-1.18.20.jar"\n' - MY_SCRIPT+=' REBUILD_CLASSPATH+=":auto-value-1.5.3.jar"\n' - MY_SCRIPT+=' javac -classpath $REBUILD_CLASSPATH Conditions.java\n' + REBUILD_CLASSPATH+=":lombok-1.18.20.jar" + REBUILD_CLASSPATH+=":auto-value-1.5.3.jar" + javac -classpath $REBUILD_CLASSPATH Conditions.java # There's an assumption that the dir ./com/claro/ was made in the tarball before this. - MY_SCRIPT+=' mv Conditions*.class com/claro\n' - MY_SCRIPT+=' jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions*.class\n' - MY_SCRIPT+=' java -classpath $CLASSPATH "${ARGS[@]}"\n' - # We insert our own exit here to short-circuit the Bazel run script's call to JAVABIN which is - # invalid on this host machine. - MY_SCRIPT+=' exit $?\n' + mv Conditions*.class com/claro + jar -uf "${RUNPATH}src/java/com/claro/claro_programs/conditions_compiled_claro_image.jar" com/claro/Conditions*.class + java -classpath $CLASSPATH "${ARGS[@]}" + EOF - # Insert MY_SCRIPT into the Bazel run script just before java gets executed. We're reusing Bazel's run - # script basically just to get a conveniently curated CLASSPATH variable generated to point to all the - # randomly scattered jar files that Bazel places throughout bazel-bin/. - sed -n "H;\${x;s/\n//;s${fancydelim}create_and_run_classpath_jar\nelse\n${fancydelim}&${MY_SCRIPT}\n${fancydelim};p;}" conditions_compiled_claro_image > conditions_compiled_claro_image2 + # Insert MY_SCRIPT into the Bazel run script just before java gets executed. We're reusing Bazel's run + # script basically just to get a conveniently curated CLASSPATH variable generated to point to all the + # randomly scattered jar files that Bazel places throughout bazel-bin/. + sed -i "s|exec \$JAVABIN.*|${MY_SCRIPT//$'\n'/\\n}|" conditions_compiled_claro_image - # I'm not good enough with sed to figure out how to avoid this extra temporary stop in a second file. Oh well. - chmod 777 conditions_compiled_claro_image - cat conditions_compiled_claro_image2 > conditions_compiled_claro_image - rm conditions_compiled_claro_image2 + chmod -R u+rw * - popd > /dev/null - ##################################################################################################### + popd + ##################################################################################################### - cp -R claro_programs "${pkg}/usr/local/bin" + cp -R claro_programs/. "${pkg}/opt/claro/programs/" - wget "https://github.com/JasonSteving99/claro-lang/releases/download/v0.1.0/claro_compiler_binary_deploy.jar" - cp claro_compiler_binary_deploy.jar "${pkg}/usr/local/bin" + wget "https://github.com/JasonSteving99/claro-lang/releases/download/v${ver}/claro_compiler_binary_deploy.jar" + cp claro_compiler_binary_deploy.jar "${pkg}/opt/claro/" setup: | - cp -R /usr/local/bin/* "${HOME}/src" + cp -R /opt/claro/programs "./" -main: "claro_programs/Conditions.claro" +main: "programs/Conditions.claro" template: | - # Thanks for trying out Claro during its early development stages! - # To learn Claro by example, check out: - # https://clarolang.com/tree/main/src/java/com/claro/claro_programs + # Thanks for trying out Claro during its early development stages! + # To learn Claro by example, check out: + # https://clarolang.com/tree/main/src/java/com/claro/claro_programs - print("Hello, world!"); - -run: | - # Skip rebuilding the entire compiler all over again and instead just use the pre-built Claro compiler jar. - chmod +rw claro_programs/Conditions.java - java -jar claro_compiler_binary_deploy.jar --java_source --silent --classname=Conditions --package=com.claro \ - < claro_programs/Conditions.claro \ - > claro_programs/Conditions.java \ - && \ - pushd claro_programs/ > /dev/null \ - && \ - ./conditions_compiled_claro_image \ - && \ - popd > /dev/null + print("Hello, world!"); repl: | - java -jar claro_compiler_binary_deploy.jar --repl --silent + java -jar /opt/claro/claro_compiler_binary_deploy.jar --repl --silent + +# Skip rebuilding the entire compiler all over again and instead just +# use the pre-built Claro compiler jar. +compile: | + java -jar /opt/claro/claro_compiler_binary_deploy.jar \ + --java_source --silent \ + --classname=Conditions --package=com.claro \ + < programs/Conditions.claro \ + > programs/Conditions.java +run: | + set -e + + cd programs + ./conditions_compiled_claro_image ||: + + java -jar /opt/claro/claro_compiler_binary_deploy.jar --repl --silent input: | - print(123 * 234); - -skip: - - runrepl + print(123 * 234); From 5a071b0aa6b97f10fa5bf706299fdd46793d73b6 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 12:40:42 -0800 Subject: [PATCH 032/123] Install tree in admin shell --- docker/admin/install.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/admin/install.bash b/docker/admin/install.bash index d64abb5..9f9fb1f 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -57,8 +57,9 @@ skopeo ssh strace sudo -tmux terraform +tmux +tree unzip uuid-runtime vim From cb2c2da32cbd761172be6efab35a22effef4f6a4 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 12:44:57 -0800 Subject: [PATCH 033/123] More --- langs/claro.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/langs/claro.yaml b/langs/claro.yaml index 6a18a54..0b88a3c 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -28,7 +28,6 @@ install: - default-jdk manual: | install -d "${pkg}/opt/claro/programs" - install -d "${pkg}/usr/local/bin" # Pull resources from the latest Claro repo release. ver="$(latest_release JasonSteving99/claro-lang | sed 's/^v//')" From 90d5ed70807e4ae65a1ec52757750f79e337eb6e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 12:48:22 -0800 Subject: [PATCH 034/123] Bump timeout --- langs/claro.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/langs/claro.yaml b/langs/claro.yaml index 0b88a3c..2f6aae4 100644 --- a/langs/claro.yaml +++ b/langs/claro.yaml @@ -103,3 +103,5 @@ run: | input: | print(123 * 234); + +timeoutFactor: 2 From bd3ebcd3e5ef61947c9ebe38804db90a98303815 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 12:49:50 -0800 Subject: [PATCH 035/123] Some formatting changes --- langs/v.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/langs/v.yaml b/langs/v.yaml index 7b7b22f..6a9ff01 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -5,7 +5,7 @@ name: "V" info: year: 2019 - desc: "V is a simple, statically-typed compiled programming language designed for building maintainable software" + desc: "Simple, statically-typed compiled programming language designed for building maintainable software" ext: - v web: @@ -27,8 +27,10 @@ install: manual: | git clone https://github.com/vlang/v.git pushd v + make - cp v ${pkg}/usr/local/bin + cp v "${pkg}/usr/local/bin/" + popd main: "main.v" @@ -44,7 +46,7 @@ format: run: | v fmt main.v input: | - fn main() + fn main() { println("Hello, world!") } From a67009d8791dd5846a9296a757f2ead402a62781 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 13:05:55 -0800 Subject: [PATCH 036/123] Get things working --- langs/v.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/langs/v.yaml b/langs/v.yaml index 6a9ff01..a8c1b11 100644 --- a/langs/v.yaml +++ b/langs/v.yaml @@ -25,18 +25,26 @@ info: install: manual: | - git clone https://github.com/vlang/v.git - pushd v + install -d "${pkg}/opt" + install -d "${pkg}/usr/local/bin" + + git clone https://github.com/vlang/v.git "${pkg}/opt/v" + pushd "${pkg}/opt/v" make - cp v "${pkg}/usr/local/bin/" + ln -s /opt/v/v "${pkg}/usr/local/bin/" + + # Force vfmt to get compiled ahead of time, otherwise this will + # happen at first invocation and fail due to lack of write + # permissions on /opt/v. + ./v fmt < /dev/null popd main: "main.v" template: | fn main() { - println('Hello, world!') + println('Hello, world!') } run: | From 5c1cf7c727a3f8604b1c63e7ba18b0863fda9434 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 22 Dec 2021 13:44:57 -0800 Subject: [PATCH 037/123] Rate-limit remote image checks --- package.json | 7 ++++++- tools/depgraph.js | 16 ++++++++++++++++ tools/docker-util.js | 45 ++++++++++++++++++++++++++------------------ yarn.lock | 5 +++++ 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index a1ebce3..600fbd8 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "parse-passwd": "^1.0.0", "prettier": "^2.3.1", "regenerator-runtime": "^0.13.7", + "semaphore": "^1.1.0", "strip-ansi": "^6.0.0", "style-loader": "^2.0.0", "uuid": "^8.3.2", @@ -41,5 +42,9 @@ "xterm": "^4.9.0", "xterm-addon-fit": "^0.4.0", "yaml": "^1.10.0" - } + }, + "$comments": [ + "limiter version pinned due to https://github.com/jhurliman/node-rate-limiter/issues/80", + "monaco-languageclient, monaco-editor, vscode-languageserver-protocol pinned because their APIs changed a bunch and Riju hasn't been updated yet" + ] } diff --git a/tools/depgraph.js b/tools/depgraph.js index dab5e45..dcadee0 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -23,6 +23,22 @@ import { import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js"; import { runCommand } from "./util.js"; +const CONCURRENCY = 1; + +async function allPromises(callables, { concurrency }) { + const queue = new PQueue({ concurrency: concurrency }); + const results = []; + for (const callable of callables) { + queue.add(async () => { + console.log("START"); + results.push(await callable()); + console.log("END"); + }); + } + await queue.onIdle(); + return results; +} + function getS3Bucket() { if (!process.env.S3_BUCKET) { throw new Error(`unset environment variable: \$S3_BUCKET`); diff --git a/tools/docker-util.js b/tools/docker-util.js index 7121d46..930b12e 100644 --- a/tools/docker-util.js +++ b/tools/docker-util.js @@ -1,5 +1,7 @@ import process from "process"; +import semaphore from "semaphore"; + import { runCommand } from "./util.js"; // Return the digest of a local image. This is the actual image @@ -41,33 +43,40 @@ export async function getRemoteRepositoryTags(repo) { ).Tags; } +const remoteImageRateLimiter = semaphore(16); + // Return the value of a label on a Docker image that is on a remote // registry. If the image or label doesn't exist, return null. You // have to pass in a list of tags on the remote repository (see // getRemoteRepositoryTags) so that we can distinguish between missing // images and network errors. export async function getRemoteImageLabel(image, label, tags) { - const [repo, tag] = image.split(":"); - let output; + await new Promise((resolve) => remoteImageRateLimiter.take(resolve)); try { - output = ( - await runCommand(`skopeo inspect docker://${image}`, { - getStdout: true, - }) - ).stdout; - } catch (err) { - if (tags.includes(tag)) { - // Tag exists, something unexpected must have gone wrong when - // running skopeo inspect. - throw err; - } else { - // Tag does not exist, that must be why skopeo inspect didn't - // work. - return null; + const [_repo, tag] = image.split(":"); + let output; + try { + output = ( + await runCommand(`skopeo inspect docker://${image}`, { + getStdout: true, + }) + ).stdout; + } catch (err) { + if (tags.includes(tag)) { + // Tag exists, something unexpected must have gone wrong when + // running skopeo inspect. + throw err; + } else { + // Tag does not exist, that must be why skopeo inspect didn't + // work. + return null; + } } + const labels = JSON.parse(output).Labels; + return (labels && labels[label]) || null; + } finally { + remoteImageRateLimiter.leave(); } - const labels = JSON.parse(output).Labels; - return (labels && labels[label]) || null; } // Return the value of $DOCKER_REPO, throwing an error if it's not set diff --git a/yarn.lock b/yarn.lock index 39039a1..607591f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3927,6 +3927,11 @@ schema-utils@^3.0.0: ajv "^6.12.5" ajv-keywords "^3.5.2" +semaphore@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa" + integrity sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA== + semver@7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" From f73e8bb5bc468390b592aec79fc24954700841b9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 23 Dec 2021 16:05:44 -0800 Subject: [PATCH 038/123] Work around more languages that are really slow --- langs/groovy.yaml | 2 ++ langs/ioke.yaml | 2 ++ langs/kotlin.yaml | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/langs/groovy.yaml b/langs/groovy.yaml index 024666d..a3da221 100644 --- a/langs/groovy.yaml +++ b/langs/groovy.yaml @@ -18,3 +18,5 @@ run: | scope: code: | x = 123 * 234; + +timeoutFactor: 4 diff --git a/langs/ioke.yaml b/langs/ioke.yaml index 709b00d..bbde50a 100644 --- a/langs/ioke.yaml +++ b/langs/ioke.yaml @@ -26,3 +26,5 @@ template: | run: | JAVA_OPTS="-Duser.home=$PWD" ioke main.ik; JAVA_OPTS="-Duser.home=$PWD" ioke + +timeoutFactor: 4 diff --git a/langs/kotlin.yaml b/langs/kotlin.yaml index b8d5608..67283a2 100644 --- a/langs/kotlin.yaml +++ b/langs/kotlin.yaml @@ -37,4 +37,4 @@ run: | JAVA_OPTS="-Duser.home=$PWD" kotlinc -script main.kts kotlinc -timeoutFactor: 2 +timeoutFactor: 4 From 4507b121a570082287a996166dbb50f6aab7c72b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 23 Dec 2021 17:43:46 -0800 Subject: [PATCH 039/123] Ugh PureScript takes too much memory or something --- langs/purescript.yaml | 72 ------------------------------------------- 1 file changed, 72 deletions(-) delete mode 100644 langs/purescript.yaml diff --git a/langs/purescript.yaml b/langs/purescript.yaml deleted file mode 100644 index a5b5c9c..0000000 --- a/langs/purescript.yaml +++ /dev/null @@ -1,72 +0,0 @@ -id: "purescript" -aliases: - - "purs" - - "pure" -name: "PureScript" - -install: - prepare: - apt: - - libtinfo5 - npm: - - purescript - - spago - apt: - - libtinfo5 - npm: - - purescript - - spago - manual: | - install -d "${pkg}/opt/purescript/skel-home" - install -d "${pkg}/opt/purescript/skel-src" - - spago init -C - rm -rf .gitignore test - sed -i 's#, "test/\*\*/\*\.purs"##' spago.dhall - - cat <<"EOF" > src/Main.purs - module Main where - - import Prelude - - import Effect (Effect) - import Effect.Console (log) - - main :: Effect Unit - main = log "Hello, world!" - EOF - - spago build - spago repl < /dev/null - - rm -rf src - - shopt -s dotglob - cp -R --preserve=timestamps * "${pkg}/opt/purescript/skel-src/" - cp -R --preserve=timestamps "${HOME}/.cache" "${pkg}/opt/purescript/skel-home/" - -setup: | - shopt -s dotglob - cp -R --preserve=timestamps /opt/purescript/skel-home/* "${HOME}/" - cp -R --preserve=timestamps /opt/purescript/skel-src/* "${PWD}/" - -repl: | - spago repl - -main: "src/Main.purs" -template: | - module Main where - - import Prelude - - import Effect (Effect) - import Effect.Console (log) - - main :: Effect Unit - main = do - log "Hello, world!" - -run: | - if spago build -n; then spago run -n; (echo 'import Prelude'; echo 'import Main') > .purs-repl; spago repl; else echo 'import Prelude' > .purs-repl; spago repl -d; fi - -timeoutFactor: 8 From 5532767e4e74493d92564c59d376f5ba2334e7d2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 23 Dec 2021 19:09:03 -0800 Subject: [PATCH 040/123] Aaaaa --- langs/scala.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/scala.yaml b/langs/scala.yaml index 78575c9..746f302 100644 --- a/langs/scala.yaml +++ b/langs/scala.yaml @@ -22,4 +22,4 @@ scope: code: | val x = 123 * 234 -timeoutFactor: 2 +timeoutFactor: 8 From 60cf5eb648b83c2216d82a95caad1d124a54a43a Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 3 Jan 2022 13:35:47 -0800 Subject: [PATCH 041/123] Add cost breakdown for Dec 2021 --- financials/2021-12/breakdown.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 financials/2021-12/breakdown.txt diff --git a/financials/2021-12/breakdown.txt b/financials/2021-12/breakdown.txt new file mode 100644 index 0000000..1edf930 --- /dev/null +++ b/financials/2021-12/breakdown.txt @@ -0,0 +1,15 @@ +Riju :: $134.10 + EC2 :: $108.76 + EBS Snapshot :: $4.40 + EBS Volume :: $27.01 + EBS Volume :: $27.01 + gp2 :: $1.02 + gp3 :: $26.00 + Instance :: $77.35 + t3.large :: $77.35 + ECR :: $5.45 + Storage :: $5.45 + ELB :: $19.77 + LCUs :: $0.09 + Load Balancer :: $19.68 + S3 :: $0.12 From ad90c5c0ccabae446f18e855e6cb74521c624e41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Jan 2022 04:42:57 +0000 Subject: [PATCH 042/123] Bump nanoid from 3.1.23 to 3.2.0 Bumps [nanoid](https://github.com/ai/nanoid) from 3.1.23 to 3.2.0. - [Release notes](https://github.com/ai/nanoid/releases) - [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md) - [Commits](https://github.com/ai/nanoid/compare/3.1.23...3.2.0) --- updated-dependencies: - dependency-name: nanoid dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 607591f..c53a585 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3211,9 +3211,9 @@ nan@^2.12.1: integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== nanoid@^3.1.23: - version "3.1.23" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.23.tgz#f744086ce7c2bc47ee0a8472574d5c78e4183a81" - integrity sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw== + version "3.2.0" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c" + integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA== nanomatch@^1.2.9: version "1.2.13" From 3aba7c43947f4666e95dd092450cada620b26e51 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 23 Jan 2022 15:54:08 -0800 Subject: [PATCH 043/123] Expose basic Prometheus metrics --- backend/server.js | 8 ++++++++ package.json | 1 + yarn.lock | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/backend/server.js b/backend/server.js index 457a679..ca968a1 100644 --- a/backend/server.js +++ b/backend/server.js @@ -5,6 +5,7 @@ import path from "path"; import express from "express"; import ws from "express-ws"; import _ from "lodash"; +import * as promClient from "prom-client"; import * as api from "./api.js"; import { aliases, langsPromise } from "./langs.js"; @@ -20,12 +21,19 @@ const analyticsTag = (process.env.ANALYTICS_TAG || "").replace( "$1" ); +promClient.collectDefaultMetrics(); + const langs = await langsPromise; const app = express(); app.set("query parser", (qs) => new URLSearchParams(qs)); app.set("view engine", "ejs"); +app.get("/metrics", async (_, res) => { + res.contentType("text/plain; version=0.0.4"); + res.send(await promClient.register.metrics()); +}); + app.get("/", (_, res) => { if (Object.keys(langs).length > 0) { res.render(path.resolve("frontend/pages/index"), { diff --git a/package.json b/package.json index 600fbd8..1e27340 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "p-queue": "^6.6.2", "parse-passwd": "^1.0.0", "prettier": "^2.3.1", + "prom-client": "^14.0.1", "regenerator-runtime": "^0.13.7", "semaphore": "^1.1.0", "strip-ansi": "^6.0.0", diff --git a/yarn.lock b/yarn.lock index c53a585..acce626 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1362,6 +1362,11 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" +bintrees@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524" + integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ= + bluebird@^3.5.5: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" @@ -3595,6 +3600,13 @@ process@^0.11.10: resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI= +prom-client@^14.0.1: + version "14.0.1" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.0.1.tgz#bdd9583e02ec95429677c0e013712d42ef1f86a8" + integrity sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w== + dependencies: + tdigest "^0.1.1" + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" @@ -4221,6 +4233,13 @@ tapable@^1.0.0, tapable@^1.1.3: resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== +tdigest@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021" + integrity sha1-Ljyyw56kSeVdHmzZEReszKRYgCE= + dependencies: + bintrees "1.0.1" + terser-webpack-plugin@^1.4.3: version "1.4.5" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz#a217aefaea330e734ffacb6120ec1fa312d6040b" From 71dcd62af44c611a58993bd1b7562068c7cecdd9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 23 Jan 2022 16:00:44 -0800 Subject: [PATCH 044/123] Put metrics on separate port --- Makefile | 8 ++++++-- backend/server.js | 16 +++++++++++----- docker/runtime/Dockerfile | 1 + 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index c2d94d7..c6a5925 100644 --- a/Makefile +++ b/Makefile @@ -62,13 +62,17 @@ endif VOLUME_MOUNT ?= $(PWD) +# http P1 ?= 6119 +# https P2 ?= 6120 +# metrics +P3 ?= 6121 ifneq (,$(EE)) -SHELL_PORTS := -p 0.0.0.0:$(P1):6119 -p 0.0.0.0:$(P2):6120 +SHELL_PORTS := -p 0.0.0.0:$(P1):6119 -p 0.0.0.0:$(P2):6120 -p 0.0.0.0:$(P3):6121 else ifneq (,$(E)) -SHELL_PORTS := -p 127.0.0.1:$(P1):6119 -p 127.0.0.1:$(P2):6120 +SHELL_PORTS := -p 127.0.0.1:$(P1):6119 -p 127.0.0.1:$(P2):6120 -p 127.0.0.1:$(P3):6121 else SHELL_PORTS := endif diff --git a/backend/server.js b/backend/server.js index ca968a1..1e7b156 100644 --- a/backend/server.js +++ b/backend/server.js @@ -15,6 +15,7 @@ import { log, privilegedTeardown } from "./util.js"; const host = process.env.HOST || "localhost"; const port = parseInt(process.env.PORT || "") || 6119; const tlsPort = parseInt(process.env.TLS_PORT || "") || 6120; +const metricsPort = parseInt(process.env.METRICS_PORT || "") || 6121; const useTLS = process.env.TLS ? true : false; const analyticsTag = (process.env.ANALYTICS_TAG || "").replace( /^'(.+)'$/, @@ -23,17 +24,18 @@ const analyticsTag = (process.env.ANALYTICS_TAG || "").replace( promClient.collectDefaultMetrics(); +const metricsApp = express(); +metricsApp.get("/metrics", async (_, res) => { + res.contentType("text/plain; version=0.0.4"); + res.send(await promClient.register.metrics()); +}); + const langs = await langsPromise; const app = express(); app.set("query parser", (qs) => new URLSearchParams(qs)); app.set("view engine", "ejs"); -app.get("/metrics", async (_, res) => { - res.contentType("text/plain; version=0.0.4"); - res.send(await promClient.register.metrics()); -}); - app.get("/", (_, res) => { if (Object.keys(langs).length > 0) { res.render(path.resolve("frontend/pages/index"), { @@ -148,3 +150,7 @@ if (useTLS) { console.log(`Listening on http://${host}:${port}`) ); } + +metricsApp.listen(metricsPort, host, () => + console.log(`Listening on http://${host}:${metricsPort}/metrics`) +); diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile index ac1118d..e659167 100644 --- a/docker/runtime/Dockerfile +++ b/docker/runtime/Dockerfile @@ -10,4 +10,5 @@ WORKDIR /src CMD ["bash"] EXPOSE 6119 EXPOSE 6120 +EXPOSE 6121 ENV HOST=0.0.0.0 From 02e52f1c883022c9efddbd6d4cc3ea57db8c9483 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sun, 30 Jan 2022 16:15:07 -0800 Subject: [PATCH 045/123] Delete unused CI scripts --- packer/ci.pkr.hcl | 48 ---------------------------------------- packer/provision-ci.bash | 35 ----------------------------- 2 files changed, 83 deletions(-) delete mode 100644 packer/ci.pkr.hcl delete mode 100755 packer/provision-ci.bash diff --git a/packer/ci.pkr.hcl b/packer/ci.pkr.hcl deleted file mode 100644 index c172a8e..0000000 --- a/packer/ci.pkr.hcl +++ /dev/null @@ -1,48 +0,0 @@ -data "amazon-ami" "ubuntu" { - filters = { - name = "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["099720109477"] -} - -locals { - timestamp = regex_replace(timestamp(), "[- TZ:]", "") -} - -source "amazon-ebs" "ubuntu" { - ami_name = "riju-ci-${local.timestamp}" - instance_type = "t3.micro" - source_ami = "${data.amazon-ami.ubuntu.id}" - ssh_username = "ubuntu" - - tag { - key = "BillingCategory" - value = "Riju" - } - - tag { - key = "BillingSubcategory" - value = "Riju:AMI" - } - - tag { - key = "Name" - value = "riju-ci-${local.timestamp}" - } -} - -build { - sources = ["source.amazon-ebs.ubuntu"] - - provisioner "file" { - destination = "/tmp/riju-init-volume" - source = "riju-init-volume" - } - - provisioner "shell" { - script = "provision-ci.bash" - } -} diff --git a/packer/provision-ci.bash b/packer/provision-ci.bash deleted file mode 100755 index d1258b1..0000000 --- a/packer/provision-ci.bash +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# I think there is a race condition related to Ubuntu wanting to do an -# automated system upgrade at boot, which causes 'apt-get update' to -# sometimes fail with an obscure error message. -sleep 5 - -mkdir /tmp/riju-work -pushd /tmp/riju-work - -export DEBIAN_FRONTEND=noninteractive - -sudo -E apt-get update -sudo -E apt-get dist-upgrade -y - -sudo -E apt-get install -y curl gnupg lsb-release - -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -E apt-key add - - -ubuntu_name="$(lsb_release -cs)" - -sudo tee -a /etc/apt/sources.list.d/custom.list >/dev/null < Date: Sun, 30 Jan 2022 16:15:31 -0800 Subject: [PATCH 046/123] Export metrics to Grafana Cloud --- packer/node-exporter.service | 13 +++++++++++++ packer/prometheus.service | 13 +++++++++++++ packer/prometheus.yaml | 19 +++++++++++++++++++ packer/promtail.yaml | 2 +- packer/provision-web.bash | 34 ++++++++++++++++++++++++++++------ packer/web.pkr.hcl | 27 +++++++++++++++++++++++++++ supervisor/src/main.go | 1 + 7 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 packer/node-exporter.service create mode 100644 packer/prometheus.service create mode 100644 packer/prometheus.yaml diff --git a/packer/node-exporter.service b/packer/node-exporter.service new file mode 100644 index 0000000..5a42d9f --- /dev/null +++ b/packer/node-exporter.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus node exporter +StartLimitBurst=5 +StartLimitIntervalSec=300 + +[Service] +Type=exec +ExecStart=node_exporter +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packer/prometheus.service b/packer/prometheus.service new file mode 100644 index 0000000..043c45b --- /dev/null +++ b/packer/prometheus.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus +StartLimitBurst=5 +StartLimitIntervalSec=300 + +[Service] +Type=exec +ExecStart=prometheus --config.file /etc/prometheus/config.yaml +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packer/prometheus.yaml b/packer/prometheus.yaml new file mode 100644 index 0000000..fbc2550 --- /dev/null +++ b/packer/prometheus.yaml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: server + static_configs: + - targets: ["localhost:6121"] + - job_name: node + static_configs: + - targets: ["localhost:9100"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + +remote_write: + - url: "https://prometheus-blocks-prod-us-central1.grafana.net/api/prom/push" + basic_auth: + username: "$GRAFANA_PROMETHEUS_USERNAME" + password: "$GRAFANA_API_KEY" diff --git a/packer/promtail.yaml b/packer/promtail.yaml index f2e1c5d..9edf4e3 100644 --- a/packer/promtail.yaml +++ b/packer/promtail.yaml @@ -7,7 +7,7 @@ positions: filename: /tmp/positions.yaml client: - url: https://72217:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push + url: https://$GRAFANA_LOKI_USERNAME:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push scrape_configs: - job_name: kernel diff --git a/packer/provision-web.bash b/packer/provision-web.bash index dac1f74..a23ddfb 100644 --- a/packer/provision-web.bash +++ b/packer/provision-web.bash @@ -77,16 +77,38 @@ if [[ -n "${GRAFANA_API_KEY:-}" ]]; then unzip promtail-linux-amd64.zip sudo cp promtail-linux-amd64 /usr/local/bin/promtail - sudo chown root:root /tmp/promtail.service /tmp/promtail.yaml + ver="$(latest_release prometheus/node_exporter | sed 's/^v//')" - sudo mkdir /etc/promtail + wget -nv "https://github.com/prometheus/node_exporter/releases/download/v${ver}/node_exporter-${ver}.linux-amd64.tar.gz" -O node_exporter.tar.gz + tar -xf node_exporter.tar.gz --strip-components=1 + sudo cp node_exporter /usr/local/bin/ + + ver="$(latest_release prometheus/prometheus | sed 's/^v//')" + + wget -nv "https://github.com/prometheus/prometheus/releases/download/v${ver}/prometheus-${ver}.linux-amd64.tar.gz" -O prometheus.tar.gz + tar -xf prometheus.tar.gz --strip-components=1 + sudo cp prometheus /usr/local/bin/ + + sudo chown root:root \ + /tmp/node-exporter.service /tmp/prometheus.service \ + /tmp/prometheus.yaml /tmp/promtail.service /tmp/promtail.yaml + + sudo mkdir /etc/prometheus /etc/promtail + sudo mv /tmp/prometheus.yaml /etc/prometheus/config.yaml sudo mv /tmp/promtail.yaml /etc/promtail/config.yaml - sudo mv /tmp/promtail.service /etc/systemd/system/ - sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" /etc/promtail/config.yaml + sudo mv /tmp/prometheus.service /tmp/promtail.service /tmp/node-exporter.service \ + /etc/systemd/system/ - sudo systemctl enable promtail + sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" \ + /etc/prometheus/config.yaml /etc/promtail/config.yaml + sudo sed -Ei "s/\\\$GRAFANA_LOKI_USERNAME/${GRAFANA_LOKI_USERNAME}/" \ + /etc/promtail/config.yaml + sudo sed -Ei "s/\\\$GRAFANA_PROMETHEUS_USERNAME/${GRAFANA_PROMETHEUS_USERNAME}/" \ + /etc/prometheus/config.yaml + + sudo systemctl enable node-exporter prometheus promtail else - sudo rm /tmp/promtail.yaml /tmp/promtail.service + sudo rm /tmp/node-exporter.service /tmp/promtail.yaml /tmp/promtail.service fi sudo userdel ubuntu -f diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index c20557a..1268e95 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -13,6 +13,16 @@ variable "analytics_tag" { default = "${env("ANALYTICS_TAG")}" } +variable "grafana_loki_username" { + type = string + default = "${env("GRAFANA_LOKI_USERNAME")}" +} + +variable "grafana_prometheus_username" { + type = string + default = "${env("GRAFANA_PROMETHEUS_USERNAME")}" +} + variable "grafana_api_key" { type = string default = "${env("GRAFANA_API_KEY")}" @@ -82,6 +92,21 @@ build { source = "docker.json" } + provisioner "file" { + destination = "/tmp/node-exporter.service" + source = "node-exporter.service" + } + + provisioner "file" { + destination = "/tmp/prometheus.service" + source = "prometheus.service" + } + + provisioner "file" { + destination = "/tmp/prometheus.yaml" + source = "prometheus.yaml" + } + provisioner "file" { destination = "/tmp/promtail.service" source = "promtail.service" @@ -117,6 +142,8 @@ build { "ADMIN_PASSWORD=${var.admin_password}", "AWS_REGION=${var.aws_region}", "ANALYTICS_TAG=${var.analytics_tag}", + "GRAFANA_LOKI_USERNAME=${var.grafana_loki_username}", + "GRAFANA_PROMETHEUS_USERNAME=${var.grafana_prometheus_username}", "GRAFANA_API_KEY=${var.grafana_api_key}", "S3_BUCKET=${var.s3_bucket}", "SENTRY_DSN=${var.sentry_dsn}", diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 2600475..7aaeeb6 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -355,6 +355,7 @@ func (sv *supervisor) reload() error { "-v", "/var/cache/riju:/var/cache/riju", "-v", "/var/run/docker.sock:/var/run/docker.sock", "-p", fmt.Sprintf("127.0.0.1:%d:6119", port), + "-p", "127.0.0.1:6121:6121", "-e", "ANALYTICS_TAG", "-e", "RIJU_DEPLOY_CONFIG", "-e", "SENTRY_DSN", From bb3a61a5cf99af84e75286d79017ed142dec1989 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 3 Feb 2022 17:40:54 -0800 Subject: [PATCH 047/123] Add financial data for Jan 2022 --- financials/2022-01/breakdown.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 financials/2022-01/breakdown.txt diff --git a/financials/2022-01/breakdown.txt b/financials/2022-01/breakdown.txt new file mode 100644 index 0000000..4a0c7a1 --- /dev/null +++ b/financials/2022-01/breakdown.txt @@ -0,0 +1,15 @@ +Riju :: $134.19 + EC2 :: $108.65 + EBS Snapshot :: $4.58 + EBS Volume :: $26.86 + EBS Volume :: $26.86 + gp2 :: $1.01 + gp3 :: $25.84 + Instance :: $77.21 + t3.large :: $77.21 + ECR :: $5.51 + Storage :: $5.51 + ELB :: $19.92 + LCUs :: $0.24 + Load Balancer :: $19.68 + S3 :: $0.11 From 2376ee641030d1e8f335e7c1f0fe410dd72d42be Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 9 Feb 2022 17:11:25 -0800 Subject: [PATCH 048/123] Allow customizing S3 config path --- Makefile | 3 ++- doc/selfhosting.md | 1 + packer/provision-web.bash | 2 ++ packer/riju.service | 1 + packer/web.pkr.hcl | 6 ++++++ supervisor/src/main.go | 7 ++++--- tools/packer-build-web.bash | 1 + 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c6a5925..4237bdb 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,10 @@ export BUILD := build/$(T)/$(L) DEB := riju-$(T)-$(L).deb S3 := s3://$(S3_BUCKET) +S3_CONFIG_PATH ?= config.json S3_DEB := $(S3)/debs/$(DEB) S3_HASH := $(S3)/hashes/riju-$(T)-$(L) -S3_CONFIG := $(S3)/config.json +S3_CONFIG := $(S3)/$(S3_CONFIG_PATH) ifneq ($(CMD),) C_CMD := -c '$(CMD)' diff --git a/doc/selfhosting.md b/doc/selfhosting.md index 43bb580..9beeab9 100644 --- a/doc/selfhosting.md +++ b/doc/selfhosting.md @@ -124,6 +124,7 @@ AMI_NAME=riju-web-20210711223158 AWS_REGION=us-west-1 DOMAIN=your.domain S3_BUCKET=yourname-riju +S3_CONFIG_PATH=config.json ``` ### AMI\_NAME diff --git a/packer/provision-web.bash b/packer/provision-web.bash index a23ddfb..281fa01 100644 --- a/packer/provision-web.bash +++ b/packer/provision-web.bash @@ -5,6 +5,7 @@ set -euo pipefail : ${ADMIN_PASSWORD} : ${AWS_REGION} : ${S3_BUCKET} +: ${S3_CONFIG_PATH} : ${SUPERVISOR_ACCESS_TOKEN} latest_release() { @@ -61,6 +62,7 @@ sudo sed -Ei 's|^#?PermitEmptyPasswords .*|PermitEmptyPasswords no|' /etc/ssh/ss sudo sed -Ei "s|\\\$AWS_REGION|${AWS_REGION}|" /etc/systemd/system/riju.service sudo sed -Ei "s|\\\$ANALYTICS_TAG|${ANALYTICS_TAG:-}|" /etc/systemd/system/riju.service sudo sed -Ei "s|\\\$S3_BUCKET|${S3_BUCKET}|" /etc/systemd/system/riju.service +sudo sed -Ei "s|\\\$S3_CONFIG_PATH|${S3_CONFIG_PATH}|" /etc/systemd/system/riju.service sudo sed -Ei "s|\\\$SENTRY_DSN|${SENTRY_DSN:-}|" /etc/systemd/system/riju.service sudo sed -Ei "s|\\\$SUPERVISOR_ACCESS_TOKEN|${SUPERVISOR_ACCESS_TOKEN}|" /etc/systemd/system/riju.service diff --git a/packer/riju.service b/packer/riju.service index c11177b..4bd3334 100644 --- a/packer/riju.service +++ b/packer/riju.service @@ -13,6 +13,7 @@ RestartSec=5 Environment=AWS_REGION=$AWS_REGION Environment=ANALYTICS_TAG=$ANALYTICS_TAG Environment=S3_BUCKET=$S3_BUCKET +Environment=S3_CONFIG_PATH=$S3_CONFIG_PATH Environment=SENTRY_DSN=$SENTRY_DSN Environment=SUPERVISOR_ACCESS_TOKEN=$SUPERVISOR_ACCESS_TOKEN diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index 1268e95..0eced82 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -33,6 +33,11 @@ variable "s3_bucket" { default = "${env("S3_BUCKET")}" } +variable "s3_config_path" { + type = string + default = "${env("S3_CONFIG_PATH")}" +} + variable "sentry_dsn" { type = string default = "${env("SENTRY_DSN_PACKER")}" @@ -146,6 +151,7 @@ build { "GRAFANA_PROMETHEUS_USERNAME=${var.grafana_prometheus_username}", "GRAFANA_API_KEY=${var.grafana_api_key}", "S3_BUCKET=${var.s3_bucket}", + "S3_CONFIG_PATH=${var.s3_config_path}", "SENTRY_DSN=${var.sentry_dsn}", "SUPERVISOR_ACCESS_TOKEN=${var.supervisor_access_token}", ] diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 7aaeeb6..b2b3ffb 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -43,8 +43,9 @@ type deploymentConfig struct { } type supervisorConfig struct { - AccessToken string `env:"SUPERVISOR_ACCESS_TOKEN,notEmpty"` - S3Bucket string `env:"S3_BUCKET,notEmpty"` + AccessToken string `env:"SUPERVISOR_ACCESS_TOKEN,notEmpty"` + S3Bucket string `env:"S3_BUCKET,notEmpty"` + S3ConfigPath string `env:"S3_CONFIG_PATH,notEmpty"` } type reloadJob struct { @@ -264,7 +265,7 @@ func (sv *supervisor) reload() error { buf := s3manager.NewWriteAtBuffer([]byte{}) if _, err := dl.Download(context.Background(), buf, &s3.GetObjectInput{ Bucket: &sv.config.S3Bucket, - Key: aws.String("config.json"), + Key: aws.String(sv.config.S3ConfigPath), }); err != nil { return err } diff --git a/tools/packer-build-web.bash b/tools/packer-build-web.bash index c0c7891..b556f71 100755 --- a/tools/packer-build-web.bash +++ b/tools/packer-build-web.bash @@ -4,6 +4,7 @@ set -euo pipefail : ${ADMIN_PASSWORD} : ${S3_BUCKET} +: ${S3_CONFIG_PATH} : ${SUPERVISOR_ACCESS_TOKEN} export AWS_REGION="${AWS_REGION:-$(aws configure get region)}" From 99cc29f0de28f5e5e4728cbf795781c887237af0 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 9 Feb 2022 17:11:31 -0800 Subject: [PATCH 049/123] Delete unused CI infrastructure --- .github/workflows/main.yml | 17 ------- tools/ci-bootstrap.bash | 10 ---- tools/ci-ec2.bash | 97 -------------------------------------- tools/ci-run.bash | 6 --- tools/ci-user-data.bash | 15 ------ 5 files changed, 145 deletions(-) delete mode 100644 .github/workflows/main.yml delete mode 100755 tools/ci-bootstrap.bash delete mode 100755 tools/ci-ec2.bash delete mode 100755 tools/ci-run.bash delete mode 100755 tools/ci-user-data.bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index a30b7ff..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Build and deploy -on: - push: - branches: - - master -concurrency: deploy -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - # - name: Build and deploy - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_REGION: us-west-1 - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # run: tools/ci-ec2.bash diff --git a/tools/ci-bootstrap.bash b/tools/ci-bootstrap.bash deleted file mode 100755 index 65fbde2..0000000 --- a/tools/ci-bootstrap.bash +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -: ${AWS_ACCESS_KEY_ID} -: ${AWS_SECRET_ACCESS_KEY} -: ${DOCKER_REPO} -: ${S3_BUCKET} - -make image shell I=ci CMD="tools/ci-run.bash" NI=1 diff --git a/tools/ci-ec2.bash b/tools/ci-ec2.bash deleted file mode 100755 index fa54cbf..0000000 --- a/tools/ci-ec2.bash +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -remote_url="$(git remote get-url origin | sed -E 's|git@github\.com:|https://github.com/|')" -commit="$(git rev-parse HEAD)" - -echo >&2 "[ci-run.bash] Fetching build parameters from SSM." - -parameters="riju-ci-ami-id riju-docker-repo-host riju-public-docker-repo-host riju-s3-bucket-name" -resp="$(aws ssm get-parameters --names ${parameters})" - -read -r ami < <(jq '.Parameters[] | select(.Name == "riju-ci-ami-id").Value' -r <<< "${resp}") -read -r docker_repo < <(jq '.Parameters[] | select(.Name == "riju-docker-repo-host").Value' -r <<< "${resp}") -read -r public_docker_repo < <(jq '.Parameters[] | select(.Name == "riju-public-docker-repo-host").Value' -r <<< "${resp}") -read -r s3_bucket < <(jq '.Parameters[] | select(.Name == "riju-s3-bucket-name").Value' -r <<< "${resp}") - -echo >&2 "[ci-run.bash] Launching EC2 instance for CI job." - -ebs_config="DeviceName=/dev/sdh,Ebs={DeleteOnTermination=true,VolumeSize=128,VolumeType=gp3}" -instance_tags="ResourceType=instance,Tags=[{Key=Name,Value=Riju CI},{Key=BillingCategory,Value=Riju},{Key=BillingSubcategory,Value=Riju:EC2:CI}]" -ebs_tags="ResourceType=volume,Tags=[{Key=Name,Value=Riju CI},{Key=BillingCategory,Value=Riju},{Key=BillingSubcategory,Value=Riju:EBS:CI}]" - -resp="$(aws ec2 run-instances \ - --image-id "${ami}" \ - --instance-type t3.2xlarge \ - --security-groups riju-deploy \ - --iam-instance-profile Name=riju-deploy \ - --instance-initiated-shutdown-behavior terminate \ - --user-data file://tools/ci-user-data.bash \ - --tag-specifications "${instance_tags}" "${ebs_tags}" \ - --block-device-mappings "${ebs_config}")" - -instance_id="$(jq '.Instances[].InstanceId' -r <<< "${resp}")" - -echo >&2 "[ci-run.bash] Waiting for instance ${instance_id} to become ready." - -success= -for i in $(seq 1 15); do - sleep 2 - resp="$(aws ec2 describe-instance-status --instance-id "${instance_id}")" - status="$(jq '.InstanceStatuses[].InstanceState.Name' -r <<< "${resp}")" - status="${status:-unknown}" - case "${status}" in - pending|unknown) ;; - running) success=yes; break ;; - * ) exit 1 ;; - esac -done - -if [[ -z "${success}}" ]]; then - exit 124 -fi - -echo >&2 "[ci-run.bash] Waiting for SSH to come online." - -success= -for i in $(seq 1 15); do - if (yes || true) | timeout 5 mssh "ubuntu@${instance_id}" true 2>/dev/null; then - success=yes - break - elif (( $# == 124 )); then - exit 1 - fi - sleep 2 -done - -if [[ -z "${success}}" ]]; then - exit 124 -fi - -echo >&2 "[ci-run.bash] Running CI remotely using EC2 Instance Connect." - -mssh "ubuntu@${instance_id}" bash <&2 "[ci-run.bash] CI completed." diff --git a/tools/ci-run.bash b/tools/ci-run.bash deleted file mode 100755 index f1c3875..0000000 --- a/tools/ci-run.bash +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -make ecr -make env CMD="dep deploy:live --registry --publish --yes" Z=xz CI=1 TEST_PATIENCE=4 TEST_CONCURRENCY=1 diff --git a/tools/ci-user-data.bash b/tools/ci-user-data.bash deleted file mode 100755 index a0e8dbe..0000000 --- a/tools/ci-user-data.bash +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ -z "${NOHUP:-}" ]]; then - NOHUP=1 nohup "$0" "$@" & -fi - -while true; do - sleep 60 - # https://unix.stackexchange.com/a/92579 - if ! sudo netstat -tnpa | grep 'ESTABLISHED.*sshd'; then - sudo shutdown -h now - fi -done From 9e80286cbe0cdecae10f5439ab8a99266c865493 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Feb 2022 21:01:59 -0800 Subject: [PATCH 050/123] Add more logs from systemd --- packer/promtail.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packer/promtail.yaml b/packer/promtail.yaml index 9edf4e3..272a2c9 100644 --- a/packer/promtail.yaml +++ b/packer/promtail.yaml @@ -39,6 +39,16 @@ scrape_configs: regex: "riju\\.service" target_label: source replacement: "supervisor" + - source_labels: + - __journal__systemd_unit + regex: "prometheus\\.service" + target_label: source + replacement: "prometheus" + - source_labels: + - __journal__systemd_unit + regex: "node-exporter\\.service" + target_label: source + replacement: "node-exporter" - source_labels: - source regex: "systemd" From d761cd7dffad9c7ead7d1e5e5ae635a1a7d6d12b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Feb 2022 21:44:48 -0800 Subject: [PATCH 051/123] Set external-labels correctly for Prometheus --- packer/prometheus.service | 2 +- packer/prometheus.yaml | 2 ++ packer/promtail.service | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packer/prometheus.service b/packer/prometheus.service index 043c45b..aa588ce 100644 --- a/packer/prometheus.service +++ b/packer/prometheus.service @@ -5,7 +5,7 @@ StartLimitIntervalSec=300 [Service] Type=exec -ExecStart=prometheus --config.file /etc/prometheus/config.yaml +ExecStart=bash -c 'EC2_INSTANCE_ID="$(curl -fsSL http://169.254.169.254/latest/meta-data/instance-id)" prometheus --config.file /etc/prometheus/config.yaml --enable-feature=expand-external-labels' Restart=always RestartSec=5 diff --git a/packer/prometheus.yaml b/packer/prometheus.yaml index fbc2550..c883fca 100644 --- a/packer/prometheus.yaml +++ b/packer/prometheus.yaml @@ -1,5 +1,7 @@ global: scrape_interval: 15s + external_labels: + node: "${EC2_INSTANCE_ID}" scrape_configs: - job_name: server diff --git a/packer/promtail.service b/packer/promtail.service index eabf709..36eba70 100644 --- a/packer/promtail.service +++ b/packer/promtail.service @@ -5,7 +5,7 @@ StartLimitIntervalSec=300 [Service] Type=exec -ExecStart=bash -c 'promtail -config.file /etc/promtail/config.yaml -client.external-labels instance="$(curl -fsSL http://169.254.169.254/latest/meta-data/instance-id)"' +ExecStart=bash -c 'promtail -config.file /etc/promtail/config.yaml -client.external-labels node="$(curl -fsSL http://169.254.169.254/latest/meta-data/instance-id)"' Restart=always RestartSec=5 From 9cde685a2527d19bcdfa9efb60209ebe86480270 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 11 Feb 2022 21:49:56 -0800 Subject: [PATCH 052/123] Remove unneeded Terraform --- tf/ami.tf | 9 --------- tf/ssm.tf | 31 ------------------------------- 2 files changed, 40 deletions(-) delete mode 100644 tf/ssm.tf diff --git a/tf/ami.tf b/tf/ami.tf index 786d29f..42894c5 100644 --- a/tf/ami.tf +++ b/tf/ami.tf @@ -6,12 +6,3 @@ data "aws_ami" "server" { values = [data.external.env.result.AMI_NAME] } } - -# data "aws_ami" "ci" { -# owners = ["self"] - -# filter { -# name = "name" -# values = [data.external.env.result.CI_AMI_NAME] -# } -# } diff --git a/tf/ssm.tf b/tf/ssm.tf deleted file mode 100644 index 3c67696..0000000 --- a/tf/ssm.tf +++ /dev/null @@ -1,31 +0,0 @@ -resource "aws_ssm_parameter" "web_ami_id" { - name = "riju-web-ami-id" - type = "String" - value = data.aws_ami.server.id - data_type = "aws:ec2:image" -} - -# resource "aws_ssm_parameter" "ci_ami_id" { -# name = "riju-ci-ami-id" -# type = "String" -# value = data.aws_ami.ci.id -# data_type = "aws:ec2:image" -# } - -resource "aws_ssm_parameter" "docker_repo" { - name = "riju-docker-repo-host" - type = "String" - value = aws_ecr_repository.riju.repository_url -} - -resource "aws_ssm_parameter" "public_docker_repo" { - name = "riju-public-docker-repo-host" - type = "String" - value = aws_ecrpublic_repository.riju.repository_uri -} - -resource "aws_ssm_parameter" "s3_bucket" { - name = "riju-s3-bucket-name" - type = "String" - value = aws_s3_bucket.riju.bucket -} From b93477132c6fbca92123e8f4e73f26ade52f046e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 09:31:19 -0800 Subject: [PATCH 053/123] Supervisor proxies metrics --- supervisor/src/main.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/supervisor/src/main.go b/supervisor/src/main.go index b2b3ffb..36f84ef 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -34,6 +34,9 @@ import ( const bluePort = 6229 const greenPort = 6230 +const blueMetricsPort = 6231 +const greenMetricsPort = 6232 + const blueName = "riju-app-blue" const greenName = "riju-app-green" @@ -339,14 +342,17 @@ func (sv *supervisor) reload() error { )) } var port int + var metricsPort int var name string var oldName string if sv.isGreen { port = bluePort + metricsPort = blueMetricsPort name = blueName oldName = greenName } else { port = greenPort + metricsPort = greenMetricsPort name = greenName oldName = blueName } @@ -356,7 +362,7 @@ func (sv *supervisor) reload() error { "-v", "/var/cache/riju:/var/cache/riju", "-v", "/var/run/docker.sock:/var/run/docker.sock", "-p", fmt.Sprintf("127.0.0.1:%d:6119", port), - "-p", "127.0.0.1:6121:6121", + "-p", fmt.Sprintf("127.0.0.1:%d:6121", metricsPort), "-e", "ANALYTICS_TAG", "-e", "RIJU_DEPLOY_CONFIG", "-e", "SENTRY_DSN", @@ -374,7 +380,7 @@ func (sv *supervisor) reload() error { } sv.status("waiting for container to start up") time.Sleep(5 * time.Second) - sv.status("checking that container is healthy") + sv.status("checking that container responds to HTTP") resp, err := http.Get(fmt.Sprintf("http://localhost:%d", port)) if err != nil { return err @@ -385,7 +391,25 @@ func (sv *supervisor) reload() error { return err } if !strings.Contains(string(body), "python") { - return errors.New("container did not appear to be healthy") + return errors.New("container did not respond successfully to HTTP") + } + sv.status("checking that container exposes metrics") + resp, err = http.Get(fmt.Sprintf("http://localhost:%d/metrics", metricsPort)) + if err != nil { + return err + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + return err + } + if !strings.Contains(string(body), "process_cpu_seconds_total") { + return errors.New("container did not expose metrics properly") + } + if sv.isGreen { + sv.status("switching from green to blue") + } else { + sv.status("switching from blue to green") } sv.isGreen = !sv.isGreen sv.status("stopping old container") From 128d26ef8e78521d420f696901ccc106879244d8 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 09:31:28 -0800 Subject: [PATCH 054/123] Reduce volume size --- tf/ec2.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/ec2.tf b/tf/ec2.tf index 9f31e58..00087d9 100644 --- a/tf/ec2.tf +++ b/tf/ec2.tf @@ -50,7 +50,7 @@ resource "aws_launch_template" "server" { device_name = "/dev/sdh" ebs { volume_type = "gp3" - volume_size = 256 + volume_size = 128 } } From a1f2a16ad3d26512493d6d1b0c4daebb012af32c Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 09:32:26 -0800 Subject: [PATCH 055/123] Update documentation also --- doc/selfhosting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/selfhosting.md b/doc/selfhosting.md index 9beeab9..f39f507 100644 --- a/doc/selfhosting.md +++ b/doc/selfhosting.md @@ -232,7 +232,7 @@ from the load balancer). to upload the finished build artifacts to ECR, which amount to about 40 GB of data transfer. If you don't have a symmetric Internet plan at home, you may need to do this on an EC2 instance instead. You can -provision one manually with at least 256 GB of disk space, install +provision one manually with at least 128 GB of disk space, install Docker, clone down Riju, copy over your `.env` file, and proceed as if you were running locally.)* From 1a7118a33c7b047aa06a569cc5751c7b9c222c26 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 09:44:15 -0800 Subject: [PATCH 056/123] Add dashboard JSON from Grafana --- grafana/dashboard.json | 902 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 902 insertions(+) create mode 100644 grafana/dashboard.json diff --git a/grafana/dashboard.json b/grafana/dashboard.json new file mode 100644 index 0000000..6353061 --- /dev/null +++ b/grafana/dashboard.json @@ -0,0 +1,902 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "iteration": 1644684886066, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 15, + "title": "Server metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum(rate(process_cpu_seconds_total{node=~\"$node\",job=\"server\"}[1m])) by (node) / count(sum(node_cpu_seconds_total{node=~\"$node\",mode=\"idle\"}) by (node, cpu)) by (node) * 100", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum(process_resident_memory_bytes{node=~\"$node\",job=\"server\"} / node_memory_MemTotal_bytes{node=~\"$node\"}) by (node) * 100", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "title": "Instance metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum(1 - rate(node_cpu_seconds_total{node=~\"$node\",mode=\"idle\"}[1m])) by (node) / count(sum(node_cpu_seconds_total{node=~\"$node\",mode=\"idle\"}) by (node, cpu)) by (node) * 100", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum(1 - node_memory_MemAvailable_bytes{node=~\"$node\"} / node_memory_MemTotal_bytes{node=~\"$node\"}) by (node) * 100", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum (rate(node_network_receive_bytes_total[1m])) by (node) / 1e6", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Network Traffic Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "sum (rate(node_network_transmit_bytes_total[1m])) by (node) / 1e3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Network Traffic Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "(1 - sum (node_filesystem_free_bytes{mountpoint=\"/\"}) by (node) / sum (node_filesystem_size_bytes{mountpoint=\"/\"}) by (node)) * 100", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Root Volume Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "exemplar": true, + "expr": "(1 - sum (node_filesystem_free_bytes{mountpoint=\"/mnt/riju\"}) by (node) / sum (node_filesystem_size_bytes{mountpoint=\"/mnt/riju\"}) by (node)) * 100", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Data Volume Disk Utilization", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 2, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Ascending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "expr": "{source=~\"$log_source\",node=~\"$node\"} | regexp \"(?P.+)\" | line_format \"{{ .node }} {{ .log }}\"", + "maxLines": 50, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Logs", + "type": "logs" + } + ], + "refresh": "5s", + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": ["i-0ec6e28b124698fc0"], + "value": ["i-0ec6e28b124698fc0"] + }, + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "definition": "label_values(node)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(node)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "loki", + "uid": "grafanacloud-logs" + }, + "definition": "label_values(source)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "log_source", + "options": [], + "query": "label_values(source)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Riju", + "uid": "mx3ZlzMnk", + "version": 28, + "weekStart": "" +} From a2e10eb116ff8d8236b830b91fec59e42f1b2ec9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 10:12:44 -0800 Subject: [PATCH 057/123] Fix some bugs in dashboard --- grafana/dashboard.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grafana/dashboard.json b/grafana/dashboard.json index 6353061..1fd06c4 100644 --- a/grafana/dashboard.json +++ b/grafana/dashboard.json @@ -22,7 +22,7 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 4, - "iteration": 1644684886066, + "iteration": 1644689175462, "links": [], "liveNow": false, "panels": [ @@ -498,7 +498,7 @@ "uid": "grafanacloud-prom" }, "exemplar": true, - "expr": "sum (rate(node_network_receive_bytes_total[1m])) by (node) / 1e6", + "expr": "sum (rate(node_network_receive_bytes_total{node=~\"$node\"}[1m])) by (node) / 1e6", "hide": false, "interval": "", "legendFormat": "", @@ -589,7 +589,7 @@ "uid": "grafanacloud-prom" }, "exemplar": true, - "expr": "sum (rate(node_network_transmit_bytes_total[1m])) by (node) / 1e3", + "expr": "sum (rate(node_network_transmit_bytes_total{node=~\"$node\"}[1m])) by (node) / 1e3", "hide": false, "interval": "", "legendFormat": "", @@ -681,7 +681,7 @@ "uid": "grafanacloud-prom" }, "exemplar": true, - "expr": "(1 - sum (node_filesystem_free_bytes{mountpoint=\"/\"}) by (node) / sum (node_filesystem_size_bytes{mountpoint=\"/\"}) by (node)) * 100", + "expr": "(1 - sum (node_filesystem_free_bytes{node=~\"$node\",mountpoint=\"/\"}) by (node) / sum (node_filesystem_size_bytes{node=~\"$node\",mountpoint=\"/\"}) by (node)) * 100", "hide": false, "interval": "", "legendFormat": "", @@ -773,7 +773,7 @@ "uid": "grafanacloud-prom" }, "exemplar": true, - "expr": "(1 - sum (node_filesystem_free_bytes{mountpoint=\"/mnt/riju\"}) by (node) / sum (node_filesystem_size_bytes{mountpoint=\"/mnt/riju\"}) by (node)) * 100", + "expr": "(1 - sum (node_filesystem_free_bytes{node=~\"$node\",mountpoint=\"/mnt/riju\"}) by (node) / sum (node_filesystem_size_bytes{node=~\"$node\",mountpoint=\"/mnt/riju\"}) by (node)) * 100", "hide": false, "interval": "", "legendFormat": "", @@ -842,9 +842,9 @@ "list": [ { "current": { - "selected": true, - "text": ["i-0ec6e28b124698fc0"], - "value": ["i-0ec6e28b124698fc0"] + "selected": false, + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "loki", @@ -866,7 +866,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": ["All"], "value": ["$__all"] }, @@ -897,6 +897,6 @@ "timezone": "", "title": "Riju", "uid": "mx3ZlzMnk", - "version": 28, + "version": 30, "weekStart": "" } From e99a537ec20220475843b89c1ad41803b6ecf973 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 10:13:07 -0800 Subject: [PATCH 058/123] Add cortextool to admin image --- docker/admin/install.bash | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 9f9fb1f..3b7b394 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -2,7 +2,12 @@ set -euxo pipefail -pushd /tmp +latest_release() { + curl -sSL "https://api.github.com/repos/$1/releases/latest" | jq -r .tag_name +} + +mkdir /tmp/riju-work +pushd /tmp/riju-work export DEBIAN_FRONTEND=noninteractive @@ -78,7 +83,11 @@ npm install -g prettier wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip unzip -q awscli.zip ./aws/install -rm -rf aws awscli.zip + +ver="$(latest_release grafana/cortex-tools | sed 's/^v//')" +wget -nv "https://github.com/grafana/cortex-tools/releases/download/v${ver}/cortextool_${ver}_linux_amd64.tar.gz" -O cortextool.tar.gz +tar -xf cortextool.tar.gz +cp cortextool /usr/local/bin/ rm -rf /var/lib/apt/lists/* @@ -87,5 +96,6 @@ tee /etc/sudoers.d/90-riju >/dev/null <<"EOF" EOF popd +rm -rf /tmp/riju-work rm "$0" From 817aa3e0fc846d2b7cab4e795e349ecaf3b2a405 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 10:14:44 -0800 Subject: [PATCH 059/123] Rename some scripts --- Makefile | 7 +++---- doc/selfhosting.md | 10 +++++----- packer/{provision-web.bash => provision.bash} | 0 packer/web.pkr.hcl | 6 +++--- 4 files changed, 11 insertions(+), 12 deletions(-) rename packer/{provision-web.bash => provision.bash} (100%) diff --git a/Makefile b/Makefile index 4237bdb..c165b03 100644 --- a/Makefile +++ b/Makefile @@ -287,11 +287,10 @@ fmt: fmt-c fmt-go fmt-python fmt-terraform fmt-web # Format all code ### Infrastructure -packer-web: supervisor # Build and publish a new webserver AMI - tools/packer-build-web.bash +packer: supervisor # Build and publish a new webserver AMI + tools/packer-build.bash + -packer-ci: # Build and publish a new CI AMI - tools/packer-build-ci.bash ### Miscellaneous diff --git a/doc/selfhosting.md b/doc/selfhosting.md index f39f507..78f7df1 100644 --- a/doc/selfhosting.md +++ b/doc/selfhosting.md @@ -111,8 +111,8 @@ API. Generate one randomly with `pwgen -s 30 1`. ## Build web AMI You'll want to run `set -a; . .env` to load in the new variables from -`.env`. Now run `make packer-web`. This will take up to 10 minutes to -build a timestamped AMI with a name like `riju-web-20210711223158`. +`.env`. Now run `make packer`. This will take up to 10 minutes to +build a timestamped AMI with a name like `riju-20210711223158`. ## Create local configuration (part 2 of 3) @@ -120,7 +120,7 @@ Add to `.env` the following contents: ``` # Terraform -AMI_NAME=riju-web-20210711223158 +AMI_NAME=riju-20210711223158 AWS_REGION=us-west-1 DOMAIN=your.domain S3_BUCKET=yourname-riju @@ -129,7 +129,7 @@ S3_CONFIG_PATH=config.json ### AMI\_NAME -This is the AMI name from `make packer-web`. +This is the AMI name from `make packer`. ### AWS\_REGION @@ -169,7 +169,7 @@ infrastructure. follow these steps:* 1. Update `.env` and make sure it is sourced (`set -a; . .env`). -2. Run `make packer-web` and get the name of the new AMI. +2. Run `make packer` and get the name of the new AMI. 3. Update it in `.env` under `AMI_NAME` and make sure the update is sourced (`set -a; . .env`). 4. Run `terraform apply`. diff --git a/packer/provision-web.bash b/packer/provision.bash similarity index 100% rename from packer/provision-web.bash rename to packer/provision.bash diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index 0eced82..9244348 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -63,7 +63,7 @@ locals { } source "amazon-ebs" "ubuntu" { - ami_name = "riju-web-${local.timestamp}" + ami_name = "riju-${local.timestamp}" instance_type = "t3.small" source_ami = "${data.amazon-ami.ubuntu.id}" ssh_username = "ubuntu" @@ -80,7 +80,7 @@ source "amazon-ebs" "ubuntu" { tag { key = "Name" - value = "riju-web-${local.timestamp}" + value = "riju-${local.timestamp}" } } @@ -155,6 +155,6 @@ build { "SENTRY_DSN=${var.sentry_dsn}", "SUPERVISOR_ACCESS_TOKEN=${var.supervisor_access_token}", ] - script = "provision-web.bash" + script = "provision.bash" } } From 7ab99e45309cc0f05854f53ea3b8e5b2e03385ec Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 18:19:04 -0800 Subject: [PATCH 060/123] Deploy AlertManager configuration --- .gitignore | 1 + Makefile | 5 ++++- docker/admin/install.bash | 1 + grafana/alertmanager.yaml | 6 ++++++ grafana/alerts.yaml | 28 ++++++++++++++++++++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 grafana/alertmanager.yaml create mode 100644 grafana/alerts.yaml diff --git a/.gitignore b/.gitignore index 9bc9590..8aeedf7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.log +*.out *.pem .env .lsp-repl-history diff --git a/Makefile b/Makefile index c165b03..0c82a3b 100644 --- a/Makefile +++ b/Makefile @@ -290,7 +290,10 @@ fmt: fmt-c fmt-go fmt-python fmt-terraform fmt-web # Format all code packer: supervisor # Build and publish a new webserver AMI tools/packer-build.bash - +deploy-alerts: # Deploy alerting configuration to Grafana Cloud + envsubst < grafana/alertmanager.yaml > grafana/alertmanager.yaml.out + cortextool rules load grafana/alerts.yaml --address=https://prometheus-blocks-prod-us-central1.grafana.net --id=$(GRAFANA_PROMETHEUS_USERNAME) --key=$(GRAFANA_API_KEY) + cortextool alertmanager load grafana/alertmanager.yaml.out --address=https://alertmanager-us-central1.grafana.net --id=$(GRAFANA_ALERTMANAGER_USERNAME) --key=$(GRAFANA_API_KEY) ### Miscellaneous diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 3b7b394..9cc3ceb 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -44,6 +44,7 @@ dctrl-tools docker-ce-cli file g++ +gettext git golang htop diff --git a/grafana/alertmanager.yaml b/grafana/alertmanager.yaml new file mode 100644 index 0000000..a106b1b --- /dev/null +++ b/grafana/alertmanager.yaml @@ -0,0 +1,6 @@ +receivers: + - name: pagerduty + pagerduty_configs: + - routing_key: "$PAGERDUTY_INTEGRATION_KEY" +route: + receiver: pagerduty diff --git a/grafana/alerts.yaml b/grafana/alerts.yaml new file mode 100644 index 0000000..87b0b67 --- /dev/null +++ b/grafana/alerts.yaml @@ -0,0 +1,28 @@ +namespace: riju +groups: + - name: riju + rules: + - alert: NodeCPUHigh + annotations: + message: "Instance {{ $labels.node }} is running close to max CPU" + expr: | + sum(1 - rate(node_cpu_seconds_total{mode="idle"}[1m])) by (node) / count(sum(node_cpu_seconds_total{mode="idle"}) by (node, cpu)) by (node) * 100 >= 80 + for: 30m + - alert: NodeMemoryHigh + annotations: + message: "Instance {{ $labels.node }} is running close to max memory" + expr: | + sum(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) by (node) * 100 >= 80 + for: 30m + - alert: RootVolumeFilling + annotations: + message: "Root volume on instance {{ $labels.node }} is close to full" + expr: | + (1 - sum (node_filesystem_free_bytes{mountpoint="/"}) by (node) / sum (node_filesystem_size_bytes{mountpoint="/"}) by (node)) * 100 + for: 30m + - alert: DataVolumeFilling + annotations: + message: "Data volume on instance {{ $labels.node }} is close to full" + expr: | + (1 - sum (node_filesystem_free_bytes{mountpoint="/mnt/riju"}) by (node) / sum (node_filesystem_size_bytes{mountpoint="/mnt/riju"}) by (node)) * 100 + for: 30m From 0bf1c8cdbdd345667975a0e88ba0f34f5ff71ed7 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 18:22:17 -0800 Subject: [PATCH 061/123] Fix alerts --- grafana/alerts.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grafana/alerts.yaml b/grafana/alerts.yaml index 87b0b67..4f62f00 100644 --- a/grafana/alerts.yaml +++ b/grafana/alerts.yaml @@ -18,11 +18,11 @@ groups: annotations: message: "Root volume on instance {{ $labels.node }} is close to full" expr: | - (1 - sum (node_filesystem_free_bytes{mountpoint="/"}) by (node) / sum (node_filesystem_size_bytes{mountpoint="/"}) by (node)) * 100 + (1 - sum (node_filesystem_free_bytes{mountpoint="/"}) by (node) / sum (node_filesystem_size_bytes{mountpoint="/"}) by (node)) * 100 >= 80 for: 30m - alert: DataVolumeFilling annotations: message: "Data volume on instance {{ $labels.node }} is close to full" expr: | - (1 - sum (node_filesystem_free_bytes{mountpoint="/mnt/riju"}) by (node) / sum (node_filesystem_size_bytes{mountpoint="/mnt/riju"}) by (node)) * 100 + (1 - sum (node_filesystem_free_bytes{mountpoint="/mnt/riju"}) by (node) / sum (node_filesystem_size_bytes{mountpoint="/mnt/riju"}) by (node)) * 100 >= 80 for: 30m From c916f917716b585a77f07b9f1f4bb4fa53a76204 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 12 Feb 2022 18:25:03 -0800 Subject: [PATCH 062/123] Delete CloudWatch --- packer/cloudwatch.json | 41 ---------- packer/provision.bash | 7 +- packer/web.pkr.hcl | 5 -- tf/cloudwatch.tf | 174 ----------------------------------------- tf/iam.tf | 58 -------------- tf/outputs.tf | 9 --- 6 files changed, 1 insertion(+), 293 deletions(-) delete mode 100644 packer/cloudwatch.json delete mode 100644 tf/cloudwatch.tf diff --git a/packer/cloudwatch.json b/packer/cloudwatch.json deleted file mode 100644 index 36ec712..0000000 --- a/packer/cloudwatch.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "agent": { - "metrics_collection_interval": 60, - "run_as_user": "root" - }, - "metrics": { - "append_dimensions": { - "ImageId": "${aws:ImageId}", - "InstanceId": "${aws:InstanceId}", - "InstanceType": "${aws:InstanceType}" - }, - "aggregation_dimensions": [ - ["RijuInstanceGroup"], - ["RijuInstanceGroup", "path"] - ], - "metrics_collected": { - "cpu": { - "append_dimensions": { - "RijuInstanceGroup": "Webserver" - }, - "measurement": ["usage_active"], - "metrics_collection_interval": 60 - }, - "disk": { - "append_dimensions": { - "RijuInstanceGroup": "Webserver" - }, - "measurement": ["used_percent"], - "metrics_collection_interval": 60, - "resources": ["/", "/mnt/riju"] - }, - "mem": { - "append_dimensions": { - "RijuInstanceGroup": "Webserver" - }, - "measurement": ["mem_used_percent"], - "metrics_collection_interval": 60 - } - } - } -} diff --git a/packer/provision.bash b/packer/provision.bash index 281fa01..a2655d6 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -44,16 +44,12 @@ sudo ./aws/install wget -nv https://s3.us-west-1.amazonaws.com/amazon-ssm-us-west-1/latest/debian_amd64/amazon-ssm-agent.deb -wget -nv https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb -sudo apt-get install -y ./amazon-cloudwatch-agent.deb - sudo chown root:root \ - /tmp/cloudwatch.json /tmp/docker.json /tmp/riju.service \ + /tmp/docker.json /tmp/riju.service \ /tmp/riju.slice /tmp/riju-init-volume /tmp/riju-supervisor sudo mv /tmp/docker.json /etc/docker/daemon.json sudo mv /tmp/riju.service /tmp/riju.slice /etc/systemd/system/ -sudo mv /tmp/cloudwatch.json /opt/aws/amazon-cloudwatch-agent/bin/config.json sudo mv /tmp/riju-init-volume /tmp/riju-supervisor /usr/local/bin/ sudo sed -Ei 's|^#?PermitRootLogin .*|PermitRootLogin no|' /etc/ssh/sshd_config @@ -69,7 +65,6 @@ sudo sed -Ei "s|\\\$SUPERVISOR_ACCESS_TOKEN|${SUPERVISOR_ACCESS_TOKEN}|" /etc/sy sudo passwd -l root sudo useradd admin -g admin -G sudo -s /usr/bin/bash -p "$(echo "${ADMIN_PASSWORD}" | mkpasswd -s)" -m -sudo amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json sudo systemctl enable riju if [[ -n "${GRAFANA_API_KEY:-}" ]]; then diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index 9244348..a7a7ff5 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -87,11 +87,6 @@ source "amazon-ebs" "ubuntu" { build { sources = ["source.amazon-ebs.ubuntu"] - provisioner "file" { - destination = "/tmp/cloudwatch.json" - source = "cloudwatch.json" - } - provisioner "file" { destination = "/tmp/docker.json" source = "docker.json" diff --git a/tf/cloudwatch.tf b/tf/cloudwatch.tf deleted file mode 100644 index 130fd9b..0000000 --- a/tf/cloudwatch.tf +++ /dev/null @@ -1,174 +0,0 @@ -resource "aws_cloudwatch_metric_alarm" "server_cpu" { - alarm_name = "riju-server-cpu-high" - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "30" - datapoints_to_alarm = "15" - metric_name = "cpu_usage_active" - namespace = "CWAgent" - period = "60" - statistic = "Average" - threshold = "70" - alarm_description = "Average CPU usage on Riju server is above 70% for 30 minutes" - ok_actions = [aws_sns_topic.riju.arn] - alarm_actions = [aws_sns_topic.riju.arn] - insufficient_data_actions = [aws_sns_topic.riju.arn] - dimensions = { - RijuInstanceGroup = "Webserver" - } - - tags = { - BillingSubcategory = "Riju:CloudWatch:Alarm" - } -} - -resource "aws_cloudwatch_metric_alarm" "server_memory" { - alarm_name = "riju-server-memory-high" - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "30" - datapoints_to_alarm = "15" - metric_name = "mem_used_percent" - namespace = "CWAgent" - period = "60" - statistic = "Average" - threshold = "70" - alarm_description = "Average memory usage on Riju server is above 70% for 30 minutes" - ok_actions = [aws_sns_topic.riju.arn] - alarm_actions = [aws_sns_topic.riju.arn] - insufficient_data_actions = [aws_sns_topic.riju.arn] - dimensions = { - RijuInstanceGroup = "Webserver" - } - - tags = { - BillingSubcategory = "Riju:CloudWatch:Alarm" - } -} - -resource "aws_cloudwatch_metric_alarm" "server_data_volume_disk_space" { - alarm_name = "riju-server-data-volume-disk-usage-high" - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "5" - datapoints_to_alarm = "5" - metric_name = "disk_used_percent" - namespace = "CWAgent" - period = "60" - statistic = "Average" - threshold = "70" - alarm_description = "Disk space usage for data volume on Riju server is above 70%" - ok_actions = [aws_sns_topic.riju.arn] - alarm_actions = [aws_sns_topic.riju.arn] - insufficient_data_actions = [aws_sns_topic.riju.arn] - dimensions = { - RijuInstanceGroup = "Webserver" - path = "/mnt/riju" - } - - tags = { - BillingSubcategory = "Riju:CloudWatch:Alarm" - } -} - -resource "aws_cloudwatch_metric_alarm" "server_root_volume_disk_space" { - alarm_name = "riju-server-root-volume-disk-usage-high" - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = "5" - datapoints_to_alarm = "5" - metric_name = "disk_used_percent" - namespace = "CWAgent" - period = "60" - statistic = "Average" - threshold = "70" - alarm_description = "Disk space usage for root volume on Riju server is above 70%" - ok_actions = [aws_sns_topic.riju.arn] - alarm_actions = [aws_sns_topic.riju.arn] - insufficient_data_actions = [aws_sns_topic.riju.arn] - dimensions = { - RijuInstanceGroup = "Webserver" - path = "/" - } - - tags = { - BillingSubcategory = "Riju:CloudWatch:Alarm" - } -} - -resource "aws_cloudwatch_dashboard" "riju" { - dashboard_name = "Riju" - dashboard_body = < Date: Sat, 12 Feb 2022 19:00:27 -0800 Subject: [PATCH 063/123] Supervisor -actually- proxies metrics --- supervisor/src/main.go | 75 ++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 36f84ef..01f4ac1 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -60,10 +60,12 @@ type reloadJob struct { type supervisor struct { config supervisorConfig - blueProxyHandler http.Handler - greenProxyHandler http.Handler - isGreen bool // blue-green deployment - deployConfigHash string + blueProxyHandler http.Handler + greenProxyHandler http.Handler + blueMetricsProxyHandler http.Handler + greenMetricsProxyHandler http.Handler + isGreen bool // blue-green deployment + deployConfigHash string awsAccountNumber string awsRegion string @@ -106,7 +108,15 @@ func (sv *supervisor) scheduleReload() string { return uuid } -func (sv *supervisor) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (sv *supervisor) serveHTTP(w http.ResponseWriter, r *http.Request, metricsPort bool) { + if metricsPort { + if sv.isGreen { + sv.greenMetricsProxyHandler.ServeHTTP(w, r) + } else { + sv.blueMetricsProxyHandler.ServeHTTP(w, r) + } + return + } if strings.HasPrefix(r.URL.Path, "/api/supervisor") { authHeader := r.Header.Get("Authorization") if authHeader == "" { @@ -477,8 +487,8 @@ func main() { } rijuInitVolume := exec.Command("riju-init-volume") - rijuInitVolume.Stdout = rijuInitVolume.Stdout - rijuInitVolume.Stderr = rijuInitVolume.Stderr + rijuInitVolume.Stdout = os.Stdout + rijuInitVolume.Stderr = os.Stderr if err := rijuInitVolume.Run(); err != nil { log.Fatalln(err) } @@ -492,6 +502,15 @@ func main() { log.Fatalln(err) } + blueMetricsUrl, err := url.Parse(fmt.Sprintf("http://localhost:%d", blueMetricsPort)) + if err != nil { + log.Fatalln(err) + } + greenMetricsUrl, err := url.Parse(fmt.Sprintf("http://localhost:%d", greenMetricsPort)) + if err != nil { + log.Fatalln(err) + } + awsCfg, err := awsConfig.LoadDefaultConfig(context.Background()) if err != nil { log.Fatalln(err) @@ -609,18 +628,38 @@ func main() { } sv := &supervisor{ - config: supervisorCfg, - blueProxyHandler: httputil.NewSingleHostReverseProxy(blueUrl), - greenProxyHandler: httputil.NewSingleHostReverseProxy(greenUrl), - isGreen: isGreen, - deployConfigHash: deployCfgHash, - s3: s3.NewFromConfig(awsCfg), - ecr: ecr.NewFromConfig(awsCfg), - awsRegion: awsCfg.Region, - awsAccountNumber: *ident.Account, - reloadJobs: map[string]*reloadJob{}, + config: supervisorCfg, + blueProxyHandler: httputil.NewSingleHostReverseProxy(blueUrl), + greenProxyHandler: httputil.NewSingleHostReverseProxy(greenUrl), + blueMetricsProxyHandler: httputil.NewSingleHostReverseProxy(blueMetricsUrl), + greenMetricsProxyHandler: httputil.NewSingleHostReverseProxy(greenMetricsUrl), + isGreen: isGreen, + deployConfigHash: deployCfgHash, + s3: s3.NewFromConfig(awsCfg), + ecr: ecr.NewFromConfig(awsCfg), + awsRegion: awsCfg.Region, + awsAccountNumber: *ident.Account, + reloadJobs: map[string]*reloadJob{}, } go sv.scheduleReload() + go func() { + log.Println("listening on http://127.0.0.1:6121/metrics") + log.Fatalln(http.ListenAndServe( + "127.0.0.1:6121", + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + sv.serveHTTP(w, r, true) + }, + ), + )) + }() log.Println("listening on http://0.0.0.0:80") - log.Fatalln(http.ListenAndServe("0.0.0.0:80", sv)) + log.Fatalln(http.ListenAndServe( + "0.0.0.0:80", + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + sv.serveHTTP(w, r, false) + }, + ), + )) } From 175b580170f0c4e339312d656a9d6ce64a54ecf4 Mon Sep 17 00:00:00 2001 From: phuchptty <13364457+phuchptty@users.noreply.github.com> Date: Sat, 26 Feb 2022 23:36:09 +0700 Subject: [PATCH 064/123] fixed: watchexec pre-release --- docker/runtime/install.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 2b7e1a5..8869bc3 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -2,8 +2,8 @@ set -euxo pipefail -latest_release() { - curl -sSL "https://api.github.com/repos/$1/releases/latest" | jq -r .tag_name +latest_watchexec_release() { + curl -sSL "https://api.github.com/repos/$1/releases" | jq -c -r '[ .[] | select( .tag_name | test("^cli-v") ) ] | first | .tag_name' } mkdir /tmp/riju-work @@ -72,7 +72,7 @@ apt-get install -y $(sed 's/#.*//' <<< "${packages}") pip3 install poetry -ver="$(latest_release watchexec/watchexec | sed 's/^cli-v//')" +ver="$(latest_watchexec_release watchexec/watchexec | sed 's/^cli-v//')" wget "https://github.com/watchexec/watchexec/releases/download/cli-v${ver}/watchexec-${ver}-x86_64-unknown-linux-gnu.deb" apt-get install -y ./watchexec-*.deb From 3a87b0dd8d8ed51a420f02dcbc776386bc05339c Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 28 Feb 2022 17:38:49 -0800 Subject: [PATCH 065/123] Change spacing --- docker/runtime/install.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/runtime/install.bash b/docker/runtime/install.bash index 8869bc3..2fce310 100755 --- a/docker/runtime/install.bash +++ b/docker/runtime/install.bash @@ -3,7 +3,7 @@ set -euxo pipefail latest_watchexec_release() { - curl -sSL "https://api.github.com/repos/$1/releases" | jq -c -r '[ .[] | select( .tag_name | test("^cli-v") ) ] | first | .tag_name' + curl -sSL "https://api.github.com/repos/$1/releases" | jq -c -r '[.[] | select(.tag_name | test("^cli-v"))] | first | .tag_name' } mkdir /tmp/riju-work From 2f4f866f24520c878ce0a56091fc4609c22202e3 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 4 Mar 2022 19:52:24 -0800 Subject: [PATCH 066/123] Add financial data for Feb 2022 --- financials/2022-02/breakdown.txt | 15 +++++++++++++++ financials/fin.py | 9 +++++++++ 2 files changed, 24 insertions(+) create mode 100644 financials/2022-02/breakdown.txt diff --git a/financials/2022-02/breakdown.txt b/financials/2022-02/breakdown.txt new file mode 100644 index 0000000..2e516c4 --- /dev/null +++ b/financials/2022-02/breakdown.txt @@ -0,0 +1,15 @@ +Riju :: $118.74 + EC2 :: $95.20 + EBS Snapshot :: $5.44 + EBS Volume :: $19.48 + EBS Volume :: $19.48 + gp2 :: $1.02 + gp3 :: $18.47 + Instance :: $70.27 + t3.large :: $70.25 + ECR :: $5.51 + Storage :: $5.51 + ELB :: $17.92 + LCUs :: $0.14 + Load Balancer :: $17.77 + S3 :: $0.11 diff --git a/financials/fin.py b/financials/fin.py index 5e74708..b1647a8 100755 --- a/financials/fin.py +++ b/financials/fin.py @@ -152,6 +152,15 @@ def classify_line_item(item, billing_month=None, full=False): and "EBS:SnapshotUsage" not in usage_type ): project = "Riju" + # Subpar tagging on my part for some testing resources. + if billing_month == "2022-02": + if service == "AmazonEC2" and resource in { + "i-04af44ee8f8238a00", + "i-0a16cf6c998e59b88", + "i-0ec6e28b124698fc0", + "i-0df1818af33ea1aa9", + }: + project = "Riju" # AWS does not let you put tags on a public ECR repository, # yippee. if service == "AmazonECRPublic" and resource.endswith("repository/riju"): From feaa029ce3896bbc75aae27bce9b0642f2f1638c Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 4 Mar 2022 19:55:16 -0800 Subject: [PATCH 067/123] Add some commentary --- financials/2022-02/breakdown.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/financials/2022-02/breakdown.txt b/financials/2022-02/breakdown.txt index 2e516c4..24a14b3 100644 --- a/financials/2022-02/breakdown.txt +++ b/financials/2022-02/breakdown.txt @@ -13,3 +13,6 @@ Riju :: $118.74 LCUs :: $0.14 Load Balancer :: $17.77 S3 :: $0.11 + +COMMENTARY: Costs are down by $16 this month because I realized that I +could cut the EBS data volume from 256 GB to 128 GB! From 21f711f90baa539cbffc0ed929083b47eb4cfcee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 02:49:14 +0000 Subject: [PATCH 068/123] Bump minimist from 1.2.5 to 1.2.6 Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index acce626..6dabe84 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3126,9 +3126,9 @@ minimatch@^3.0.4: brace-expansion "^1.1.7" minimist@^1.2.0, minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + version "1.2.6" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" + integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== mississippi@^3.0.0: version "3.0.0" From 0f81dccd7058edd7ef1d808b5c0fd232c1a1be5b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 4 Apr 2022 17:33:36 -0700 Subject: [PATCH 069/123] Add financial data for Mar 2022 --- financials/2022-03/breakdown.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 financials/2022-03/breakdown.txt diff --git a/financials/2022-03/breakdown.txt b/financials/2022-03/breakdown.txt new file mode 100644 index 0000000..35515bc --- /dev/null +++ b/financials/2022-03/breakdown.txt @@ -0,0 +1,15 @@ +Riju :: $122.42 + EC2 :: $96.90 + EBS Snapshot :: $5.70 + EBS Volume :: $13.91 + EBS Volume :: $13.91 + gp2 :: $1.01 + gp3 :: $12.90 + Instance :: $77.28 + t3.large :: $77.28 + ECR :: $5.54 + Storage :: $5.54 + ELB :: $19.87 + LCUs :: $0.19 + Load Balancer :: $19.68 + S3 :: $0.11 From 6494543e16da7cf61e770d5ca4cffd4cac243fae Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 7 May 2022 15:52:09 -0700 Subject: [PATCH 070/123] Financial data for Apr 2022 --- financials/2022-04/breakdown.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 financials/2022-04/breakdown.txt diff --git a/financials/2022-04/breakdown.txt b/financials/2022-04/breakdown.txt new file mode 100644 index 0000000..82c0b36 --- /dev/null +++ b/financials/2022-04/breakdown.txt @@ -0,0 +1,15 @@ +Riju :: $119.48 + EC2 :: $94.60 + EBS Snapshot :: $5.70 + EBS Volume :: $13.91 + EBS Volume :: $13.91 + gp2 :: $1.01 + gp3 :: $12.90 + Instance :: $74.98 + t3.large :: $74.98 + ECR :: $5.57 + Storage :: $5.57 + ELB :: $19.20 + LCUs :: $0.16 + Load Balancer :: $19.04 + S3 :: $0.11 From 0605798c41be02f63ef083dcae0a1d5aefdc0a15 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 9 May 2022 09:52:11 -0700 Subject: [PATCH 071/123] Update to reflect IP transfer --- .github/FUNDING.yml | 16 +++++++++------- CONTRIBUTING.md | 6 +++--- LICENSE.md | 3 ++- Makefile | 2 +- SECURITY.md | 4 ++-- cli/go.mod | 2 +- doc/local.md | 2 +- doc/what-languages.md | 9 +++++---- financials/2021-08/breakdown.txt | 10 +++++----- financials/pyproject.toml | 2 +- frontend/pages/index.ejs | 6 ++++-- langs/kalyn.yaml | 2 +- supervisor/go.mod | 2 +- tools/generate-build-script.js | 2 +- 14 files changed, 37 insertions(+), 31 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index f56f222..d5cdb45 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,8 +1,10 @@ -github: raxod502 -patreon: riju -ko_fi: riju_codes -liberapay: riju +github: radian-software +patreon: radiansoftware +ko_fi: radiansoftware +liberapay: radian-software custom: - - https://paypal.me/rijucodes - - https://cash.app/$RijuCodes - - https://venmo.com/code?user_id=3335527067549696598 + - https://www.paypal.com/donate/?hosted_button_id=SYF48KFJ95FPA + # Cash App is not currently supported because their sign-up flow is + # broken (verification emails for new accounts are not sent). I have + # submitted a support ticket with them. + - https://account.venmo.com/u/radian-software diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6009098..a274323 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,6 +8,6 @@ * [Deploying your own instance of Riju](doc/selfhosting.md) If you'd like to request a new language, head to the [language support -meta-issue](https://github.com/raxod502/riju/issues/24) and add a -comment. Of course, if you actually want it to be added anytime soon, -you should submit a pull request :) +meta-issue](https://github.com/radian-software/riju/issues/24) and add +a comment. Of course, if you actually want it to be added anytime +soon, you should submit a pull request :) diff --git a/LICENSE.md b/LICENSE.md index ad907d4..bac17b1 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,7 @@ # MIT License -Copyright (c) 2020 Radon Rosborough +Copyright (c) 2020–2022 [Radian LLC](https://radian.codes) and +contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 0c82a3b..da23920 100644 --- a/Makefile +++ b/Makefile @@ -218,7 +218,7 @@ lsp: # L= : Run LSP REPL for language or custom command line ### Fetch artifacts from registries -PUBLIC_DOCKER_REPO_PULL ?= public.ecr.aws/raxod502/riju +PUBLIC_DOCKER_REPO_PULL ?= public.ecr.aws/radian-software/riju sync-ubuntu: # Pull Riju Ubuntu image from public Docker registry docker pull $(PUBLIC_DOCKER_REPO_PULL):ubuntu diff --git a/SECURITY.md b/SECURITY.md index 7d94501..12d2165 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,8 +1,8 @@ # Reporting a security issue Please contact me at -[radon.neon@gmail.com](mailto:radon.neon@gmail.com) if you find any -way to: +[security+riju@radian.codes](mailto:security+riju@radian.codes) if you +find any way to: * Take down Riju without using a large number of concurrent sessions. * View or interfere with another user's session. diff --git a/cli/go.mod b/cli/go.mod index 3aa8505..8586943 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -1,4 +1,4 @@ -module github.com/raxod502/riju/cli +module github.com/radian-software/riju/cli go 1.16 diff --git a/doc/local.md b/doc/local.md index 2218ed9..f1bf43b 100644 --- a/doc/local.md +++ b/doc/local.md @@ -13,7 +13,7 @@ hesitate to open an issue! Clone locally: ``` -$ git clone https://github.com/raxod502/riju.git +$ git clone https://github.com/radian-software/riju.git $ cd riju ``` diff --git a/doc/what-languages.md b/doc/what-languages.md index 50cb764..5180217 100644 --- a/doc/what-languages.md +++ b/doc/what-languages.md @@ -49,9 +49,9 @@ requirements: because it only runs on macOS, and [Docker](https://www.docker.com/) is out because it can't be run inside Docker (without the `--privileged` flag, which has unacceptable security drawbacks; see - [#29](https://github.com/raxod502/riju/issues/29)). Note, however, - that many Windows-based languages can be used successfully via - [Mono](https://www.mono-project.com/) or + [#29](https://github.com/radian-software/riju/issues/29)). Note, + however, that many Windows-based languages can be used successfully + via [Mono](https://www.mono-project.com/) or [Wine](https://www.winehq.org/), such as [Cmd](https://en.wikipedia.org/wiki/Cmd.exe), [C#](https://en.wikipedia.org/wiki/C_Sharp_(programming_language)), @@ -60,7 +60,8 @@ requirements: Here are some explicit *non-requirements*: * *Language must be well-known.* Nope, I'll be happy to add your pet - project; after all, [Kalyn](https://github.com/raxod502/kalyn) and + project; after all, + [Kalyn](https://github.com/radian-software/kalyn) and [Ink](https://github.com/thesephist/ink) are already supported. * *Language must be useful.* I have no objection to adding everything on the esolangs wiki, if there are interpreters/compilers available. diff --git a/financials/2021-08/breakdown.txt b/financials/2021-08/breakdown.txt index b389475..556d25a 100644 --- a/financials/2021-08/breakdown.txt +++ b/financials/2021-08/breakdown.txt @@ -18,8 +18,8 @@ Riju :: $58.75 S3 :: $0.13 COMMENTARY: I think we could save on ELB costs by migrating to Lambda; -see https://github.com/raxod502/riju/issues/93 for that. Otherwise, -the main important thing to note about this month is that I had part -of the infrastructure spun down for a significant part of it, as per -https://riju.statuspage.io/ (Aug 1 through Aug 16). So costs are -liable to increase next month now that we are in normal operation. +see https://github.com/radian-software/riju/issues/93 for that. +Otherwise, the main important thing to note about this month is that I +had part of the infrastructure spun down for a significant part of it, +as per https://riju.statuspage.io/ (Aug 1 through Aug 16). So costs +are liable to increase next month now that we are in normal operation. diff --git a/financials/pyproject.toml b/financials/pyproject.toml index cb626d9..2cce438 100644 --- a/financials/pyproject.toml +++ b/financials/pyproject.toml @@ -2,7 +2,7 @@ name = "riju-financials" version = "0.1.0" description = "Financial data for Riju hosting" -authors = ["Radon Rosborough "] +authors = ["Radian LLC "] [tool.poetry.dependencies] python = "^3.9" diff --git a/frontend/pages/index.ejs b/frontend/pages/index.ejs index ac45a26..fd38f97 100644 --- a/frontend/pages/index.ejs +++ b/frontend/pages/index.ejs @@ -25,9 +25,11 @@

Created by - Radon Rosborough. + Radon Rosborough + and maintained by + Radian LLC. Check out the project - on GitHub. + on GitHub.

<% } else { %> diff --git a/langs/kalyn.yaml b/langs/kalyn.yaml index 46793c4..e22c425 100644 --- a/langs/kalyn.yaml +++ b/langs/kalyn.yaml @@ -9,7 +9,7 @@ install: install -d "${pkg}/opt/kalyn" install -d "${pkg}/usr/local/bin" - git clone https://github.com/raxod502/kalyn.git + git clone https://github.com/radian-software/kalyn.git pushd kalyn stack build kalyn cp "$(stack exec which kalyn)" "${pkg}/usr/local/bin/" diff --git a/supervisor/go.mod b/supervisor/go.mod index 3021b2a..51b41c6 100644 --- a/supervisor/go.mod +++ b/supervisor/go.mod @@ -1,4 +1,4 @@ -module github.com/raxod502/riju/supervisor +module github.com/radian-software/riju/supervisor go 1.16 diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index 3952b72..329177a 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -312,7 +312,7 @@ sudo --preserve-env=DEBIAN_FRONTEND apt-get update`); Package: riju-${isShared ? "shared" : "lang"}-${id} Version: \$(date +%s%3N) Architecture: amd64 -Maintainer: Radon Rosborough +Maintainer: Radian LLC Description: The ${name} ${ isShared ? "shared dependency" : "language" } packaged for Riju From 4f99a456aa053c8db4d3a6100b1d210c76253903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 09:53:25 -0700 Subject: [PATCH 072/123] Bump ejs from 3.1.6 to 3.1.7 (#159) Bumps [ejs](https://github.com/mde/ejs) from 3.1.6 to 3.1.7. - [Release notes](https://github.com/mde/ejs/releases) - [Changelog](https://github.com/mde/ejs/blob/main/CHANGELOG.md) - [Commits](https://github.com/mde/ejs/compare/v3.1.6...v3.1.7) --- updated-dependencies: - dependency-name: ejs dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 71 ++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 1e27340..f6163ba 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "css-loader": "^5.0.1", "debounce": "^1.2.0", "docker-file-parser": "^1.0.5", - "ejs": "^3.1.5", + "ejs": "^3.1.7", "express": "^4.17.1", "express-ws": "^4.0.0", "file-loader": "^6.2.0", diff --git a/yarn.lock b/yarn.lock index 6dabe84..96ad992 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1175,6 +1175,13 @@ ansi-styles@^3.2.1: dependencies: color-convert "^1.9.0" +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + anymatch@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb" @@ -1259,10 +1266,10 @@ async-lock@^1.2.6: resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.3.0.tgz#0fba111bea8b9693020857eba4f9adca173df3e5" integrity sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg== -async@0.9.x: - version "0.9.2" - resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d" - integrity sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0= +async@^3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" + integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g== atob@^2.1.2: version "2.1.2" @@ -1592,7 +1599,7 @@ caniuse-lite@^1.0.30001219: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz#4b7783661515b8e7151fc6376cfd97f0e427b9e5" integrity sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw== -chalk@^2.0.0, chalk@^2.4.2: +chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -1601,6 +1608,14 @@ chalk@^2.0.0, chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chalk@^4.0.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -1687,11 +1702,23 @@ color-convert@^1.9.0: dependencies: color-name "1.1.3" +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + colorette@^1.2.1, colorette@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" @@ -2001,12 +2028,12 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= -ejs@^3.1.5: - version "3.1.6" - resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.6.tgz#5bfd0a0689743bb5268b3550cceeebbc1702822a" - integrity sha512-9lt9Zse4hPucPkoP7FHDF0LQAlGyF9JVpnClFLFH3aSSbxmyoqINRpp/9wePWJTUl4KOQwRL72Iw3InHPDkoGw== +ejs@^3.1.7: + version "3.1.7" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.7.tgz#c544d9c7f715783dd92f0bddcf73a59e6962d006" + integrity sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw== dependencies: - jake "^10.6.1" + jake "^10.8.5" electron-to-chromium@^1.3.723: version "1.3.752" @@ -2487,6 +2514,11 @@ has-flag@^3.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + has-symbols@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423" @@ -2835,13 +2867,13 @@ isobject@^3.0.0, isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= -jake@^10.6.1: - version "10.8.2" - resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.2.tgz#ebc9de8558160a66d82d0eadc6a2e58fbc500a7b" - integrity sha512-eLpKyrfG3mzvGE2Du8VoPbeSkRry093+tyNjdYaBbJS9v17knImYGNXQCUV0gLxQtF82m3E8iRb/wdSQZLoq7A== +jake@^10.8.5: + version "10.8.5" + resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" + integrity sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw== dependencies: - async "0.9.x" - chalk "^2.4.2" + async "^3.2.3" + chalk "^4.0.2" filelist "^1.0.1" minimatch "^3.0.4" @@ -4228,6 +4260,13 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + tapable@^1.0.0, tapable@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" From e872b40ef646fea3757d3068fc591e9f90117ba2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 9 May 2022 10:01:44 -0700 Subject: [PATCH 073/123] Venmo does not work lmao --- .github/FUNDING.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index d5cdb45..a0e4507 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -7,4 +7,9 @@ custom: # Cash App is not currently supported because their sign-up flow is # broken (verification emails for new accounts are not sent). I have # submitted a support ticket with them. - - https://account.venmo.com/u/radian-software + # + # Venmo is not currently supported because it is impossible to + # create a new business account when one has been created at any + # point in the past, even if it has been deleted. I have reached out + # to Venmo support and they have confirmed there is no supported way + # to use Venmo going forward, and suggested I use PayPal instead. From 8b139151c83402537bd70b2ca120ea6da19796c1 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 9 May 2022 10:01:50 -0700 Subject: [PATCH 074/123] Update donations verbiage --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1a74c56..9cb1e88 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,14 @@ Service uptime available at . ## Is it free? -Riju will always be free for everyone. I pay for the hosting costs -myself. +Riju will always be free for everyone. I pay for the hosting costs out +of the business account of Radian LLC, which is funded by donations +and my personal savings. If you would like to help keep Riju online +and see more projects like it, there are a few donation methods +available in the "Sponsor this project" sidebar on GitHub. -A number of people have asked me if they can donate to help keep Riju -online. In response, I have set up a few methods, which you can see in -the "Sponsor this project" sidebar on GitHub. All donations will be -used solely to cover hosting costs, and any surplus will be donated to -the [Electronic Frontier Foundation](https://www.eff.org/). +All financial records for Radian LLC are made [publicly +available](https://github.com/radian-software/financials). ## Is it safe? From fb6948df727d541d1e03ac44514b1efdfe720e3f Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 9 May 2022 13:50:57 -0700 Subject: [PATCH 075/123] Fix various things --- Makefile | 2 +- packer/prometheus.yaml | 2 +- packer/promtail.yaml | 2 +- packer/provision.bash | 4 ++++ packer/web.pkr.hcl | 12 ++++++++++++ tools/packer-build-ci.bash | 5 ----- tools/{packer-build-web.bash => packer-build.bash} | 0 7 files changed, 19 insertions(+), 8 deletions(-) delete mode 100755 tools/packer-build-ci.bash rename tools/{packer-build-web.bash => packer-build.bash} (100%) diff --git a/Makefile b/Makefile index da23920..5b9d507 100644 --- a/Makefile +++ b/Makefile @@ -292,7 +292,7 @@ packer: supervisor # Build and publish a new webserver AMI deploy-alerts: # Deploy alerting configuration to Grafana Cloud envsubst < grafana/alertmanager.yaml > grafana/alertmanager.yaml.out - cortextool rules load grafana/alerts.yaml --address=https://prometheus-blocks-prod-us-central1.grafana.net --id=$(GRAFANA_PROMETHEUS_USERNAME) --key=$(GRAFANA_API_KEY) + cortextool rules load grafana/alerts.yaml --address=https://$(GRAFANA_PROMETHEUS_HOSTNAME) --id=$(GRAFANA_PROMETHEUS_USERNAME) --key=$(GRAFANA_API_KEY) cortextool alertmanager load grafana/alertmanager.yaml.out --address=https://alertmanager-us-central1.grafana.net --id=$(GRAFANA_ALERTMANAGER_USERNAME) --key=$(GRAFANA_API_KEY) ### Miscellaneous diff --git a/packer/prometheus.yaml b/packer/prometheus.yaml index c883fca..c29f345 100644 --- a/packer/prometheus.yaml +++ b/packer/prometheus.yaml @@ -15,7 +15,7 @@ scrape_configs: - targets: ["localhost:9090"] remote_write: - - url: "https://prometheus-blocks-prod-us-central1.grafana.net/api/prom/push" + - url: "$GRAFANA_PROMETHEUS_HOSTNAME" basic_auth: username: "$GRAFANA_PROMETHEUS_USERNAME" password: "$GRAFANA_API_KEY" diff --git a/packer/promtail.yaml b/packer/promtail.yaml index 272a2c9..eaec8ba 100644 --- a/packer/promtail.yaml +++ b/packer/promtail.yaml @@ -7,7 +7,7 @@ positions: filename: /tmp/positions.yaml client: - url: https://$GRAFANA_LOKI_USERNAME:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push + url: https://$GRAFANA_LOKI_USERNAME:$GRAFANA_API_KEY@$GRAFANA_LOKI_HOSTNAME/api/prom/push scrape_configs: - job_name: kernel diff --git a/packer/provision.bash b/packer/provision.bash index a2655d6..9a7e2c7 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -98,8 +98,12 @@ if [[ -n "${GRAFANA_API_KEY:-}" ]]; then sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" \ /etc/prometheus/config.yaml /etc/promtail/config.yaml + sudo sed -Ei "s/\\\$GRAFANA_LOKI_HOSTNAME/${GRAFANA_LOKI_HOSTNAME}/" \ + /etc/promtail/config.yaml sudo sed -Ei "s/\\\$GRAFANA_LOKI_USERNAME/${GRAFANA_LOKI_USERNAME}/" \ /etc/promtail/config.yaml + sudo sed -Ei "s/\\\$GRAFANA_PROMETHEUS_HOSTNAME/${GRAFANA_PROMETHEUS_HOSTNAME}/" \ + /etc/prometheus/config.yaml sudo sed -Ei "s/\\\$GRAFANA_PROMETHEUS_USERNAME/${GRAFANA_PROMETHEUS_USERNAME}/" \ /etc/prometheus/config.yaml diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index a7a7ff5..afd6b66 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -13,11 +13,21 @@ variable "analytics_tag" { default = "${env("ANALYTICS_TAG")}" } +variable "grafana_loki_hostname" { + type = string + default = "${env("GRAFANA_LOKI_HOSTNAME")}" +} + variable "grafana_loki_username" { type = string default = "${env("GRAFANA_LOKI_USERNAME")}" } +variable "grafana_prometheus_hostname" { + type = string + default = "${env("GRAFANA_PROMETHEUS_HOSTNAME")}" +} + variable "grafana_prometheus_username" { type = string default = "${env("GRAFANA_PROMETHEUS_USERNAME")}" @@ -142,7 +152,9 @@ build { "ADMIN_PASSWORD=${var.admin_password}", "AWS_REGION=${var.aws_region}", "ANALYTICS_TAG=${var.analytics_tag}", + "GRAFANA_LOKI_HOSTNAME=${var.grafana_loki_hostname}", "GRAFANA_LOKI_USERNAME=${var.grafana_loki_username}", + "GRAFANA_PROMETHEUS_HOSTNAME=${var.grafana_prometheus_hostname}", "GRAFANA_PROMETHEUS_USERNAME=${var.grafana_prometheus_username}", "GRAFANA_API_KEY=${var.grafana_api_key}", "S3_BUCKET=${var.s3_bucket}", diff --git a/tools/packer-build-ci.bash b/tools/packer-build-ci.bash deleted file mode 100755 index 415a3e4..0000000 --- a/tools/packer-build-ci.bash +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -packer build ci.pkr.hcl diff --git a/tools/packer-build-web.bash b/tools/packer-build.bash similarity index 100% rename from tools/packer-build-web.bash rename to tools/packer-build.bash From afe6df3cdcf8557db3ff112000f5676741b673b9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 9 May 2022 15:41:39 -0700 Subject: [PATCH 076/123] Migrate financials to radian-software org --- financials/2021-07/breakdown.txt | 61 ------ financials/2021-08/breakdown.txt | 25 --- financials/2021-09/breakdown.txt | 27 --- financials/2021-10/breakdown.txt | 18 -- financials/2021-11/breakdown.txt | 17 -- financials/2021-12/breakdown.txt | 15 -- financials/2022-01/breakdown.txt | 15 -- financials/2022-02/breakdown.txt | 18 -- financials/2022-03/breakdown.txt | 15 -- financials/2022-04/breakdown.txt | 15 -- financials/README.md | 14 +- financials/fin.py | 319 ------------------------------- financials/poetry.lock | 135 ------------- financials/pyproject.toml | 16 -- 14 files changed, 2 insertions(+), 708 deletions(-) delete mode 100644 financials/2021-07/breakdown.txt delete mode 100644 financials/2021-08/breakdown.txt delete mode 100644 financials/2021-09/breakdown.txt delete mode 100644 financials/2021-10/breakdown.txt delete mode 100644 financials/2021-11/breakdown.txt delete mode 100644 financials/2021-12/breakdown.txt delete mode 100644 financials/2022-01/breakdown.txt delete mode 100644 financials/2022-02/breakdown.txt delete mode 100644 financials/2022-03/breakdown.txt delete mode 100644 financials/2022-04/breakdown.txt delete mode 100755 financials/fin.py delete mode 100644 financials/poetry.lock delete mode 100644 financials/pyproject.toml diff --git a/financials/2021-07/breakdown.txt b/financials/2021-07/breakdown.txt deleted file mode 100644 index a788bb5..0000000 --- a/financials/2021-07/breakdown.txt +++ /dev/null @@ -1,61 +0,0 @@ -Riju :: $169.46 - CloudWatch :: $34.80 - EC2 :: $107.01 - Data Transfer :: $0.68 - EBS Snapshot :: $5.45 - EBS Volume :: $46.40 - EBS Volume :: $46.40 - gp2 :: $11.61 - gp3 :: $34.78 - Instance :: $54.48 - t2.small :: $0.04 - t3 :: $0.08 - t3.2xlarge :: $29.80 - t3.medium :: $14.77 - t3.small :: $9.78 - ECR :: $7.31 - Data Transfer :: $3.29 - Storage :: $4.02 - ELB :: $20.05 - Data Transfer :: $0.31 - LCUs :: $0.06 - Load Balancer :: $19.68 - S3 :: $0.29 - -COMMENTARY: This month was a disaster because AWS makes it really hard -to understand what exactly is going to run up your bill. - -The most egregious thing here is CloudWatch. It turns out that if you -follow the official documentation for how to set up a CloudWatch alarm -on disk space for your EC2 instance, the default configuration has SSM -Agent creating a metric for *every* filesystem mounted on your -instance, which is actually one or more per Docker container, so I -actually had like multiple tens of thousands of metrics being shipped -to CloudWatch, which is expensive. I fixed this for August, bringing -CloudWatch costs to be effectively zero. - -We have some charges for a t3.medium, this is before I scaled the -server down to t3.small. The charges for that instance are also higher -than you'd expect because I was originally running two of them before -scaling it down for a singleton because I realized I was out of my -depth. - -We had a couple gp2 volumes (more expensive) before I migrated -everything to gp3. EBS costs are generally quite high here because not -only did I previously have two instances serving traffic, but I also -had a dev server. Each of those three instances had to have the full -256 GB data volume to store language images, which was ridiculously -expensive. I'm planning on keeping Riju as a singleton for a while -because of this issue, and relying on vertical scaling until that -becomes no longer feasible. The persistent dev server will be replaced -by a transient CI instance that can be spun up to do large rebuild -operations, mitigating EBS costs. - -t3.2xlarge is the dev server, this is mostly just tough luck since I -did need to spend a lot of time building and rebuilding language -images and those hours add up. Hopefully that won't be as much of an -issue going forward now that the infrastructure is more stable and we -can hopefully get away without a dev server in general. But -fundamentally you can't do builds on your local laptop without a -symmetric Internet plan because you need to upload like 100 GB for a -full rebuild. diff --git a/financials/2021-08/breakdown.txt b/financials/2021-08/breakdown.txt deleted file mode 100644 index 556d25a..0000000 --- a/financials/2021-08/breakdown.txt +++ /dev/null @@ -1,25 +0,0 @@ -Riju :: $58.75 - EC2 :: $32.26 - Data Transfer :: $0.04 - EBS Snapshot :: $1.67 - EBS Volume :: $18.46 - EBS Volume :: $18.46 - gp2 :: $0.69 - gp3 :: $17.77 - Instance :: $12.09 - t3.small :: $12.09 - ECR :: $6.42 - Data Transfer :: $1.38 - Storage :: $5.05 - ELB :: $19.93 - Data Transfer :: $0.18 - LCUs :: $0.06 - Load Balancer :: $19.68 - S3 :: $0.13 - -COMMENTARY: I think we could save on ELB costs by migrating to Lambda; -see https://github.com/radian-software/riju/issues/93 for that. -Otherwise, the main important thing to note about this month is that I -had part of the infrastructure spun down for a significant part of it, -as per https://riju.statuspage.io/ (Aug 1 through Aug 16). So costs -are liable to increase next month now that we are in normal operation. diff --git a/financials/2021-09/breakdown.txt b/financials/2021-09/breakdown.txt deleted file mode 100644 index a767444..0000000 --- a/financials/2021-09/breakdown.txt +++ /dev/null @@ -1,27 +0,0 @@ -Riju :: $81.55 - EC2 :: $57.02 - Data Transfer :: $0.02 - EBS Snapshot :: $1.97 - EBS Volume :: $26.82 - EBS Volume :: $26.82 - gp2 :: $1.01 - gp3 :: $25.81 - Instance :: $28.21 - t3.medium :: $19.01 - t3.small :: $9.21 - ECR :: $5.09 - Storage :: $5.09 - ELB :: $19.32 - Data Transfer :: $0.22 - LCUs :: $0.06 - Load Balancer :: $19.04 - S3 :: $0.12 - -COMMENTARY: We're starting to look pretty stable from month to month. -Naturally the costs are higher because we were operating the -infrastructure for the entire month this time, instead of being down -for half of it, but I think this cost is about what we should expect -to see going forward until changes are made. - -I did realize, by the way, that we can't use Lambda to replace the -ELB, because that wouldn't support websockets. Oh well. diff --git a/financials/2021-10/breakdown.txt b/financials/2021-10/breakdown.txt deleted file mode 100644 index e7b0510..0000000 --- a/financials/2021-10/breakdown.txt +++ /dev/null @@ -1,18 +0,0 @@ -Riju :: $106.77 - EC2 :: $81.38 - Data Transfer :: $0.03 - EBS Snapshot :: $2.36 - EBS Volume :: $28.57 - EBS Volume :: $28.57 - gp2 :: $1.07 - gp3 :: $27.49 - Instance :: $50.43 - t3.large :: $23.05 - t3.medium :: $27.38 - ECR :: $5.14 - Storage :: $5.14 - ELB :: $20.14 - Data Transfer :: $0.38 - LCUs :: $0.07 - Load Balancer :: $19.68 - S3 :: $0.11 diff --git a/financials/2021-11/breakdown.txt b/financials/2021-11/breakdown.txt deleted file mode 100644 index 3462f7d..0000000 --- a/financials/2021-11/breakdown.txt +++ /dev/null @@ -1,17 +0,0 @@ -Riju :: $133.50 - EC2 :: $108.22 - Data Transfer :: $0.02 - EBS Snapshot :: $3.79 - EBS Volume :: $27.58 - EBS Volume :: $27.58 - gp2 :: $1.04 - gp3 :: $26.55 - Instance :: $76.82 - t3.large :: $76.82 - ECR :: $5.34 - Storage :: $5.34 - ELB :: $19.83 - Data Transfer :: $0.69 - LCUs :: $0.10 - Load Balancer :: $19.04 - S3 :: $0.11 diff --git a/financials/2021-12/breakdown.txt b/financials/2021-12/breakdown.txt deleted file mode 100644 index 1edf930..0000000 --- a/financials/2021-12/breakdown.txt +++ /dev/null @@ -1,15 +0,0 @@ -Riju :: $134.10 - EC2 :: $108.76 - EBS Snapshot :: $4.40 - EBS Volume :: $27.01 - EBS Volume :: $27.01 - gp2 :: $1.02 - gp3 :: $26.00 - Instance :: $77.35 - t3.large :: $77.35 - ECR :: $5.45 - Storage :: $5.45 - ELB :: $19.77 - LCUs :: $0.09 - Load Balancer :: $19.68 - S3 :: $0.12 diff --git a/financials/2022-01/breakdown.txt b/financials/2022-01/breakdown.txt deleted file mode 100644 index 4a0c7a1..0000000 --- a/financials/2022-01/breakdown.txt +++ /dev/null @@ -1,15 +0,0 @@ -Riju :: $134.19 - EC2 :: $108.65 - EBS Snapshot :: $4.58 - EBS Volume :: $26.86 - EBS Volume :: $26.86 - gp2 :: $1.01 - gp3 :: $25.84 - Instance :: $77.21 - t3.large :: $77.21 - ECR :: $5.51 - Storage :: $5.51 - ELB :: $19.92 - LCUs :: $0.24 - Load Balancer :: $19.68 - S3 :: $0.11 diff --git a/financials/2022-02/breakdown.txt b/financials/2022-02/breakdown.txt deleted file mode 100644 index 24a14b3..0000000 --- a/financials/2022-02/breakdown.txt +++ /dev/null @@ -1,18 +0,0 @@ -Riju :: $118.74 - EC2 :: $95.20 - EBS Snapshot :: $5.44 - EBS Volume :: $19.48 - EBS Volume :: $19.48 - gp2 :: $1.02 - gp3 :: $18.47 - Instance :: $70.27 - t3.large :: $70.25 - ECR :: $5.51 - Storage :: $5.51 - ELB :: $17.92 - LCUs :: $0.14 - Load Balancer :: $17.77 - S3 :: $0.11 - -COMMENTARY: Costs are down by $16 this month because I realized that I -could cut the EBS data volume from 256 GB to 128 GB! diff --git a/financials/2022-03/breakdown.txt b/financials/2022-03/breakdown.txt deleted file mode 100644 index 35515bc..0000000 --- a/financials/2022-03/breakdown.txt +++ /dev/null @@ -1,15 +0,0 @@ -Riju :: $122.42 - EC2 :: $96.90 - EBS Snapshot :: $5.70 - EBS Volume :: $13.91 - EBS Volume :: $13.91 - gp2 :: $1.01 - gp3 :: $12.90 - Instance :: $77.28 - t3.large :: $77.28 - ECR :: $5.54 - Storage :: $5.54 - ELB :: $19.87 - LCUs :: $0.19 - Load Balancer :: $19.68 - S3 :: $0.11 diff --git a/financials/2022-04/breakdown.txt b/financials/2022-04/breakdown.txt deleted file mode 100644 index 82c0b36..0000000 --- a/financials/2022-04/breakdown.txt +++ /dev/null @@ -1,15 +0,0 @@ -Riju :: $119.48 - EC2 :: $94.60 - EBS Snapshot :: $5.70 - EBS Volume :: $13.91 - EBS Volume :: $13.91 - gp2 :: $1.01 - gp3 :: $12.90 - Instance :: $74.98 - t3.large :: $74.98 - ECR :: $5.57 - Storage :: $5.57 - ELB :: $19.20 - LCUs :: $0.16 - Load Balancer :: $19.04 - S3 :: $0.11 diff --git a/financials/README.md b/financials/README.md index a33bfa0..22d0263 100644 --- a/financials/README.md +++ b/financials/README.md @@ -1,14 +1,4 @@ # Riju financials -This directory has a Python script that can download and analyze -billing data from AWS to determine how much Riju actually costs. This -information is then made publicly available in per-month -subdirectories here; for some months with unusual charges I've added -commentary to explain what was going on. - -This information is then imported into [Riju's master budgeting -spreadsheet](https://docs.google.com/spreadsheets/d/15Us9KLXaJ6B1lNhrM6GV6JmmeKqNc8NNeTnaWiAhozw/edit?usp=sharing) -which compares spending to donations in order to determine whether we -are making a profit (we are not...). Once we start making a profit we -can start donating to the EFF as promised, or scale up Riju's -infrastructure to support more users for free. +This data has all moved to +[radian-software/financials](https://github.com/radian-software/financials). diff --git a/financials/fin.py b/financials/fin.py deleted file mode 100755 index b1647a8..0000000 --- a/financials/fin.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import collections -import csv -import decimal -import gzip -import io -import json -import logging -import os -import pathlib -import re -import sys -from urllib.parse import urlparse - -import boto3 - -logging.basicConfig(level=logging.INFO) - -ROOT = pathlib.Path(__file__).parent - - -def die(msg): - raise AssertionError(msg) - - -def get_csv(year, month, force_download=False): - target_dir = ROOT / f"{year}-{month:02d}" - logging.info(f"Using base directory {target_dir}") - target_dir.mkdir(exist_ok=True) - latest_csv = target_dir / "latest.csv" - if force_download or not latest_csv.exists(): - try: - latest_csv.unlink() - except FileNotFoundError: - pass - s3 = boto3.client("s3") - o = urlparse(os.environ["BILLING_REPORTS_URL"], allow_fragments=False) - assert o.scheme == "s3" - bucket = o.netloc - base_prefix = o.path.strip("/") + "/" - report_name = base_prefix.rstrip("/").split("/")[-1] - logging.info(f"List s3://{bucket}/{base_prefix}") - month_prefixes = [ - elt["Prefix"] - for elt in s3.list_objects_v2( - Bucket=bucket, Prefix=f"{base_prefix}", Delimiter="/" - )["CommonPrefixes"] - ] - if not month_prefixes: - die("no report prefixes found") - expected_month_prefix = f"{base_prefix}{year}{month:02d}" - matching_month_prefixes = [ - p for p in month_prefixes if p.startswith(expected_month_prefix) - ] - if not matching_month_prefixes: - die(f"no report prefix for the specified month ({expected_month_prefix})") - if len(matching_month_prefixes) > 1: - die(f"multiple matching report prefixes: {repr(matching_month_prefixes)}") - (month_prefix,) = matching_month_prefixes - stream = io.BytesIO() - manifest_path = f"{month_prefix}{report_name}-Manifest.json" - logging.info(f"Download s3://{bucket}/{manifest_path} in-memory") - s3.download_fileobj(bucket, manifest_path, stream) - manifest = json.loads(stream.getvalue()) - (report_path,) = manifest["reportKeys"] - if not report_path.endswith(".csv.gz"): - die(f"unexpected report extension in {report_path}") - logging.info(f"Get metadata for s3://{bucket}/{report_path}") - basename = s3.head_object(Bucket=bucket, Key=report_path)[ - "LastModified" - ].strftime("%Y-%m-%d") - logging.info( - f"Download s3://{bucket}/{report_path} to {target_dir.relative_to(ROOT)}/{basename}.csv.gz" - ) - s3.download_file(bucket, report_path, f"{target_dir}/{basename}.csv.gz") - logging.info(f"Decompress {basename}.csv.gz") - with gzip.open(f"{target_dir}/{basename}.csv.gz") as f_read: - with open(f"{target_dir}/{basename}.csv", "wb") as f_write: - while chunk := f_read.read(1024): - f_write.write(chunk) - latest_csv.symlink_to(f"{basename}.csv") - return latest_csv - - -def read_csv(csv_path): - rows = [] - with open(csv_path) as f: - reader = csv.reader(f) - header = next(reader) - for row in reader: - rows.append(dict((key, val) for (key, val) in zip(header, row) if val)) - return rows - - -def get_tax_key(item): - service = item["lineItem/ProductCode"] - usage_type = item["lineItem/UsageType"] - if "DataTransfer" in usage_type: - service = "AWSDataTransfer" - return (service, usage_type) - - -def embed_taxes(items): - tax_items = collections.defaultdict(list) - usage_items = collections.defaultdict(list) - for item in items: - item_type = item["lineItem/LineItemType"] - if item_type == "Tax": - tax_items[get_tax_key(item)].append(item) - elif item_type == "Usage": - usage_items[get_tax_key(item)].append(item) - else: - die(f"unexpected line item type {repr(item_type)}") - for key in tax_items: - if key not in usage_items: - die(f"tax for {repr(key)} but no usage for that key") - tax_cost = sum(item["lineItem/UnblendedCost"] for item in tax_items[key]) - usage_cost = sum(item["lineItem/UnblendedCost"] for item in usage_items[key]) - tax_multiplier = (tax_cost + usage_cost) / usage_cost - for item in usage_items[key]: - item["lineItem/UnblendedCost"] *= tax_multiplier - return [item for group in usage_items.values() for item in group] - - -def classify_line_item(item, billing_month=None, full=False): - service = item["lineItem/ProductCode"] - usage_type = item["lineItem/UsageType"] - operation = item.get("lineItem/Operation") - resource = item.get("lineItem/ResourceId") - project = item.get("resourceTags/user:BillingCategory") - # In 2021-07, the first month that I was using AWS resources for - # Riju in a nontrivial capacity, I had subpar billing - # observability, so a lot of the resources aren't tagged - # correctly. So for that month specifically, I'm hacking in a - # couple of heuristics to tag the resources after the fact based - # on what I know about my usage of AWS. - if billing_month == "2021-07": - if resource and "riju" in resource.lower(): - project = "Riju" - elif resource and "shallan" in resource.lower(): - project = "Shallan" - elif resource and "veidt" in resource.lower(): - project = "Veidt" - elif service == "AmazonCloudWatch": - project = "Riju" - elif ( - service == "AmazonEC2" - and resource != "i-077884b74aba86bac" - and "ElasticIP:IdleAddress" not in usage_type - and "EBS:SnapshotUsage" not in usage_type - ): - project = "Riju" - # Subpar tagging on my part for some testing resources. - if billing_month == "2022-02": - if service == "AmazonEC2" and resource in { - "i-04af44ee8f8238a00", - "i-0a16cf6c998e59b88", - "i-0ec6e28b124698fc0", - "i-0df1818af33ea1aa9", - }: - project = "Riju" - # AWS does not let you put tags on a public ECR repository, - # yippee. - if service == "AmazonECRPublic" and resource.endswith("repository/riju"): - project = "Riju" - category = [ - "Uncategorized", - service, - usage_type, - operation or "(no operation)", - resource or "(no resource)", - ] - if not full: - if service == "AmazonS3": - category = ["S3"] - elif service == "AmazonSNS": - category = ["SNS"] - elif service in ("AmazonECR", "AmazonECRPublic"): - category = ["ECR"] - if "DataTransfer" in usage_type: - category.append("Data Transfer") - elif "TimedStorage" in usage_type: - category.append("Storage") - else: - category.extend( - [ - "Uncategorized", - usage_type, - operation or "(no operation)", - resource or "(no resource)", - ] - ) - elif service == "AmazonEC2": - category = ["EC2"] - if "ElasticIP:IdleAddress" in usage_type: - category.append("EIP") - # Apparently tags on EIPs are ignored for billing - # purposes, so we just have to know what we were using - # them for. (Leaving them uncategorized for 2021-07 - # though.) - if billing_month != "2021-07": - project = "Corona" - elif "EBS:VolumeUsage" in usage_type: - category.append("EBS Volume") - category.extend(["EBS Volume", re.sub(r"^.+\.", "", usage_type)]) - elif "EBS:SnapshotUsage" in usage_type: - category.append("EBS Snapshot") - elif ( - "DataTransfer" in usage_type - or "In-Bytes" in usage_type - or "Out-Bytes" in usage_type - ): - category.append("Data Transfer") - elif "BoxUsage" in usage_type or "CPUCredits" in usage_type: - category.extend(["Instance", re.sub(r"^.+:", "", usage_type)]) - else: - category.extend( - [ - "Uncategorized", - usage_type, - operation or "(no operation)", - resource or "(no resource)", - ] - ) - elif service == "AWSELB": - category = ["ELB"] - if "DataTransfer" in usage_type: - category.append("Data Transfer") - elif "LCUUsage" in usage_type: - category.append("LCUs") - elif "LoadBalancerUsage": - category.append("Load Balancer") - else: - category.extend( - [ - "Uncategorized", - usage_type, - operation or "(no operation)", - resource or "(no resource)", - ] - ) - elif service == "AmazonCloudWatch": - category = ["CloudWatch"] - elif service == "awskms": - category = ["KMS"] - if not project: - category.extend( - [ - usage_type, - operation or "(no operation)", - resource or "(no resource)", - ] - ) - return [project or "Uncategorized", *category] - - -def add_to_taxonomy(taxonomy, category, item): - if category: - categories = taxonomy.setdefault("categories", {}) - add_to_taxonomy(categories.setdefault(category[0], {}), category[1:], item) - else: - taxonomy.setdefault("items", []).append(item) - taxonomy.setdefault("cost", 0) - taxonomy["cost"] += float(item["lineItem/UnblendedCost"]) - - -def uncategorized_last(key): - return (key == "Uncategorized", key) - - -def print_taxonomy(taxonomy, indent="", file=sys.stdout): - cost = taxonomy["cost"] - categories = taxonomy.get("categories", {}) - for category in sorted(categories, key=uncategorized_last): - subtaxonomy = categories[category] - cost = subtaxonomy["cost"] - if cost < 0.01: - continue - print(f"{indent}{category} :: ${cost:.2f}", file=file) - print_taxonomy(subtaxonomy, indent=indent + " ", file=file) - - -def classify_costs(csv_path, **kwargs): - all_items = [item for item in read_csv(csv_path)] - items = [] - for item in all_items: - cost = item["lineItem/UnblendedCost"] - if cost and float(cost): - items.append({**item, "lineItem/UnblendedCost": float(cost)}) - taxonomy = {} - for item in embed_taxes(items): - add_to_taxonomy(taxonomy, ["AWS", *classify_line_item(item, **kwargs)], item) - return taxonomy - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("date") - parser.add_argument("-f", "--force-download", action="store_true") - parser.add_argument("-w", "--write", action="store_true") - args = parser.parse_args() - year, month = map(int, args.date.split("-")) - billing_month = f"{year}-{month:02d}" - csv_path = get_csv(year, month, force_download=args.force_download) - taxonomy = classify_costs(csv_path, billing_month=billing_month) - print_taxonomy(taxonomy) - if args.write: - riju_taxonomy = taxonomy["categories"]["AWS"] - riju_taxonomy["categories"] = {"Riju": riju_taxonomy["categories"]["Riju"]} - target_dir = ROOT / f"{year}-{month:02d}" - with open(target_dir / "breakdown.txt", "w") as f: - print_taxonomy(riju_taxonomy, file=f) - - -if __name__ == "__main__": - main() - sys.exit(0) diff --git a/financials/poetry.lock b/financials/poetry.lock deleted file mode 100644 index c9ec56f..0000000 --- a/financials/poetry.lock +++ /dev/null @@ -1,135 +0,0 @@ -[[package]] -name = "boto3" -version = "1.18.23" -description = "The AWS SDK for Python" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -botocore = ">=1.21.23,<1.22.0" -jmespath = ">=0.7.1,<1.0.0" -s3transfer = ">=0.5.0,<0.6.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "botocore" -version = "1.21.23" -description = "Low-level, data-driven core of boto 3." -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -jmespath = ">=0.7.1,<1.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = ">=1.25.4,<1.27" - -[package.extras] -crt = ["awscrt (==0.11.24)"] - -[[package]] -name = "jmespath" -version = "0.10.0" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-dotenv" -version = "0.19.0" -description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.extras] -cli = ["click (>=5.0)"] - -[[package]] -name = "s3transfer" -version = "0.5.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "urllib3" -version = "1.26.6" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" - -[package.extras] -brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.9" -content-hash = "170b0bcf9f0ae12c4c9e1daa195ecdb39585494414b88e53e3da72916eb52c51" - -[metadata.files] -boto3 = [ - {file = "boto3-1.18.23-py3-none-any.whl", hash = "sha256:1b08ace99e7b92965780e5ce759430ad62b7b7e037560bc772f9a8789f4f36d2"}, - {file = "boto3-1.18.23.tar.gz", hash = "sha256:31cc69e665f773390c4c17ce340d2420e45fbac51d46d945cc4a58d483ec5da6"}, -] -botocore = [ - {file = "botocore-1.21.23-py3-none-any.whl", hash = "sha256:3877d69e0b718b786f1696cd04ddbdb3a57aef6adb0239a29aa88754489849a4"}, - {file = "botocore-1.21.23.tar.gz", hash = "sha256:d0146d31dbc475942b578b47dd5bcf94d18fbce8c6d2ce5f12195e005de9b754"}, -] -jmespath = [ - {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, - {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -python-dotenv = [ - {file = "python-dotenv-0.19.0.tar.gz", hash = "sha256:f521bc2ac9a8e03c736f62911605c5d83970021e3fa95b37d769e2bbbe9b6172"}, - {file = "python_dotenv-0.19.0-py2.py3-none-any.whl", hash = "sha256:aae25dc1ebe97c420f50b81fb0e5c949659af713f31fdb63c749ca68748f34b1"}, -] -s3transfer = [ - {file = "s3transfer-0.5.0-py3-none-any.whl", hash = "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803"}, - {file = "s3transfer-0.5.0.tar.gz", hash = "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, -] diff --git a/financials/pyproject.toml b/financials/pyproject.toml deleted file mode 100644 index 2cce438..0000000 --- a/financials/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[tool.poetry] -name = "riju-financials" -version = "0.1.0" -description = "Financial data for Riju hosting" -authors = ["Radian LLC "] - -[tool.poetry.dependencies] -python = "^3.9" -boto3 = "^1.18.23" -python-dotenv = "^0.19.0" - -[tool.poetry.dev-dependencies] - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" From e46f140597e1402a89109ad4795df8067653903c Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 11 May 2022 11:13:32 -0700 Subject: [PATCH 077/123] Change GH Sponsors back until org acct is approved --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a0e4507..bbe6a8e 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,4 @@ -github: radian-software +github: raxod502 patreon: radiansoftware ko_fi: radiansoftware liberapay: radian-software From 01bc9661a1f824d5bc9c73a1d34d07dd7a3b4f25 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 12 May 2022 12:53:45 -0700 Subject: [PATCH 078/123] New GH Sponsors org This reverts commit e46f140597e1402a89109ad4795df8067653903c. --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index bbe6a8e..a0e4507 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,4 @@ -github: raxod502 +github: radian-software patreon: radiansoftware ko_fi: radiansoftware liberapay: radian-software From a539645bb251428471bd6dfeb9ed1146b0c88601 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 7 Jun 2022 16:38:16 -0700 Subject: [PATCH 079/123] Add Cash App --- .github/FUNDING.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a0e4507..8c8ceca 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -4,10 +4,7 @@ ko_fi: radiansoftware liberapay: radian-software custom: - https://www.paypal.com/donate/?hosted_button_id=SYF48KFJ95FPA - # Cash App is not currently supported because their sign-up flow is - # broken (verification emails for new accounts are not sent). I have - # submitted a support ticket with them. - # + - https://cash.app/$RadianSoftware # Venmo is not currently supported because it is impossible to # create a new business account when one has been created at any # point in the past, even if it has been deleted. I have reached out From 460af21c3e93f6c839d8217ab862f9b3a195a39d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 1 Jul 2022 21:39:50 -0700 Subject: [PATCH 080/123] Remove IPAccounting as it has no effect Ref: https://github.com/systemd/systemd/issues/9513 Not sure what the intended use case of this configuration parameter is but it appears to be a known limitation that it does not work for what we need it to do, so remove. --- packer/riju.slice | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packer/riju.slice b/packer/riju.slice index c3e9eca..b5a7769 100644 --- a/packer/riju.slice +++ b/packer/riju.slice @@ -21,8 +21,3 @@ MemorySwapMax=0 # this space to user code. TasksAccounting=true TasksMax=400000 - -# Attempt to deny access to EC2 Instance Metadata service from user -# code. -IPAccounting=true -IPAddressDeny=169.254.169.254 From 51cda5866b38ee992633aebc7069cab2819b24d5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Fri, 1 Jul 2022 22:06:52 -0700 Subject: [PATCH 081/123] Deny access to network due to abuse reports --- system/src/riju-system-privileged.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 91f2025..b3bf6a6 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -240,6 +240,13 @@ void session(char *uuid, char *lang, char *imageHash) "4000", "--cgroup-parent", "riju.slice", + // Deny access to outside networking for now in order to limit + // abuse, as we've received abuse reports from AWS. We should + // be able to remove this (and indeed we'll *want* to, in + // order to support package installation) by replacing it with + // a more fine-grained network control such as limiting + // outbound bandwidth. + "--network=none", "--label", "riju.category=user-session", "--label", @@ -250,6 +257,7 @@ void session(char *uuid, char *lang, char *imageHash) (char *)sentinel_bash, NULL, }; + execvp(argv[0], argv); die("execvp failed"); } From edc07acdac4c98524c49d7211c5edbc781a11304 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Jul 2022 16:52:02 -0700 Subject: [PATCH 082/123] Bump terser from 4.8.0 to 4.8.1 (#160) Bumps [terser](https://github.com/terser/terser) from 4.8.0 to 4.8.1. - [Release notes](https://github.com/terser/terser/releases) - [Changelog](https://github.com/terser/terser/blob/master/CHANGELOG.md) - [Commits](https://github.com/terser/terser/commits) --- updated-dependencies: - dependency-name: terser dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index 96ad992..212fcd0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1514,9 +1514,9 @@ browserslist@^4.16.6: node-releases "^1.1.71" buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + version "1.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== buffer-xor@^1.0.3: version "1.0.3" @@ -4138,9 +4138,9 @@ source-map-resolve@^0.5.0: urix "^0.1.0" source-map-support@~0.5.12: - version "0.5.19" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" - integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== dependencies: buffer-from "^1.0.0" source-map "^0.6.0" @@ -4295,9 +4295,9 @@ terser-webpack-plugin@^1.4.3: worker-farm "^1.7.0" terser@^4.1.2: - version "4.8.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.0.tgz#63056343d7c70bb29f3af665865a46fe03a0df17" - integrity sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw== + version "4.8.1" + resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.1.tgz#a00e5634562de2239fd404c649051bf6fc21144f" + integrity sha512-4GnLC0x667eJG0ewJTa6z/yXrbLGv80D9Ru6HIpCQmO+Q4PfEtBFi0ObSckqwL6VyQv/7ENJieXHo2ANmdQwgw== dependencies: commander "^2.20.0" source-map "~0.6.1" From 62d88a2a11ddf1b3ad3b4ee08f2565df63a5c74f Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Sat, 3 Sep 2022 19:21:07 -0700 Subject: [PATCH 083/123] oh god --- docker/app/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 9a80730..c9d8ebc 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,4 +1,5 @@ -FROM riju:ubuntu AS build +# temporary hack +FROM ubuntu:20.04 AS build COPY docker/app/install-build.bash /tmp/ RUN /tmp/install-build.bash From 05566e3da4f2f0a6f959d44ab7c53b09409aaad9 Mon Sep 17 00:00:00 2001 From: RedstoneRadiant <76220359+TheRedstoneRadiant@users.noreply.github.com> Date: Thu, 22 Sep 2022 18:45:48 -0400 Subject: [PATCH 084/123] Update statuspage link (#162) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9cb1e88..f75870e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ or compiling [INTERCAL](https://en.wikipedia.org/wiki/INTERCAL) code. Check it out at ! -Service uptime available at . +Service uptime available at . ## Is it free? From c80a7acb89eeacb67289bee99a9b70adbee3fe54 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 08:53:04 -0700 Subject: [PATCH 085/123] Use LTS Ubuntu for admin image --- docker/admin/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/admin/Dockerfile b/docker/admin/Dockerfile index 1981d6d..c104e17 100644 --- a/docker/admin/Dockerfile +++ b/docker/admin/Dockerfile @@ -1,4 +1,5 @@ -FROM ubuntu:21.04 +# EOL: April 2027 +FROM ubuntu:22.04 COPY docker/admin/install.bash /tmp/ RUN /tmp/install.bash From 5fa070185aa6da7a77941569c2d97a5476d265a3 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 09:50:10 -0700 Subject: [PATCH 086/123] Remove top-level references to Depgraph hashing --- Makefile | 27 +++++-------------- tools/build-lang-image.js | 49 +++------------------------------- tools/generate-build-script.js | 3 +-- 3 files changed, 10 insertions(+), 69 deletions(-) diff --git a/Makefile b/Makefile index 5b9d507..23a3914 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ DEB := riju-$(T)-$(L).deb S3 := s3://$(S3_BUCKET) S3_CONFIG_PATH ?= config.json S3_DEB := $(S3)/debs/$(DEB) -S3_HASH := $(S3)/hashes/riju-$(T)-$(L) S3_CONFIG := $(S3)/$(S3_CONFIG_PATH) ifneq ($(CMD),) @@ -52,13 +51,10 @@ image: # I= [L=] [NC=1] : Build a Docker image ifeq ($(I),lang) @: $${L} node tools/build-lang-image.js --lang $(L) -else ifeq ($(I),ubuntu) - docker pull ubuntu:21.04 - hash="$$(docker inspect ubuntu:21.04 -f '{{ .Id }}' | sha1sum | awk '{ print $$1 }')"; echo "FROM ubuntu:21.04" | docker build --label riju.image-hash="$${hash}" -t riju:$(I) - else ifneq (,$(filter $(I),admin ci)) docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) else - hash="$$(node tools/hash-dockerfile.js $(I) | grep .)"; docker build . -f docker/$(I)/Dockerfile -t riju:$(I) --label riju.image-hash="$${hash}" $(NO_CACHE) + docker build . -f docker/$(I)/Dockerfile -t riju:$(I) $(NO_CACHE) endif VOLUME_MOUNT ?= $(PWD) @@ -86,25 +82,22 @@ else LANG_TAG := $(I) endif -IMAGE_HASH := "$$(docker inspect riju:$(LANG_TAG) -f '{{ index .Config.Labels "riju.image-hash" }}')" -WITH_IMAGE_HASH := -e RIJU_IMAGE_HASH=$(IMAGE_HASH) - shell: # I= [L=] [E[E]=1] [P1|P2=] [CMD="..."] : Launch Docker image with shell @: $${I} ifneq (,$(filter $(I),admin ci)) @mkdir -p $(HOME)/.aws $(HOME)/.docker $(HOME)/.ssh $(HOME)/.terraform.d - docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/cache/riju/.aws -v $(HOME)/.docker:/var/cache/riju/.docker -v $(HOME)/.ssh:/var/cache/riju/.ssh -v $(HOME)/.terraform.d:/var/cache/riju/.terraform.d -e NI -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_REPO -e PUBLIC_DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) --network host riju:$(I) $(BASH_CMD) + docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock -v $(HOME)/.aws:/var/cache/riju/.aws -v $(HOME)/.docker:/var/cache/riju/.docker -v $(HOME)/.ssh:/var/cache/riju/.ssh -v $(HOME)/.terraform.d:/var/cache/riju/.terraform.d -e NI -e AWS_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e DOCKER_REPO -e PUBLIC_DOCKER_REPO -e S3_BUCKET -e DOMAIN -e VOLUME_MOUNT=$(VOLUME_MOUNT) $(SHELL_PORTS) $(SHELL_ENV) --network host riju:$(I) $(BASH_CMD) else ifeq ($(I),app) - docker run $(IT_ARG) --rm --hostname $(I) -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run $(IT_ARG) --rm --hostname $(I) -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) else ifneq (,$(filter $(I),base lang)) ifeq ($(I),lang) @: $${L} endif - docker run $(IT_ARG) --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(LANG_TAG) $(BASH_CMD) + docker run $(IT_ARG) --rm --hostname $(LANG_TAG) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(LANG_TAG) $(BASH_CMD) else ifeq ($(I),runtime) - docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src -v /var/cache/riju:/var/cache/riju -v /var/run/docker.sock:/var/run/docker.sock $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) else - docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) $(WITH_IMAGE_HASH) riju:$(I) $(BASH_CMD) + docker run $(IT_ARG) --rm --hostname $(I) -v $(VOLUME_MOUNT):/src $(SHELL_PORTS) $(SHELL_ENV) riju:$(I) $(BASH_CMD) endif ecr: # Authenticate to ECR (temporary credentials) @@ -242,21 +235,13 @@ undeploy: # Pull latest deployment config from S3 push: # I= : Push Riju image to Docker registry @: $${I} $${DOCKER_REPO} - docker tag riju:$(I) $(DOCKER_REPO):$(I)-$(IMAGE_HASH) - docker push $(DOCKER_REPO):$(I)-$(IMAGE_HASH) -ifeq ($(I),ubuntu) - docker tag riju:$(I) $(PUBLIC_DOCKER_REPO):$(I) - docker push $(PUBLIC_DOCKER_REPO):$(I) -endif docker tag riju:$(I) $(DOCKER_REPO):$(I) docker push $(DOCKER_REPO):$(I) upload: # L= T= : Upload .deb to S3 @: $${L} $${T} $${S3_BUCKET} tools/ensure-deb-compressed.bash - aws s3 rm --recursive $(S3_HASH) aws s3 cp $(BUILD)/$(DEB) $(S3_DEB) - hash="$$(dpkg-deb -f $(BUILD)/$(DEB) Riju-Script-Hash | grep .)"; aws s3 cp - "$(S3_HASH)/$${hash}" < /dev/null deploy-config: # Generate deployment config file node tools/generate-deploy-config.js diff --git a/tools/build-lang-image.js b/tools/build-lang-image.js index 3888f19..831d38a 100644 --- a/tools/build-lang-image.js +++ b/tools/build-lang-image.js @@ -1,15 +1,10 @@ -import crypto from "crypto"; -import { promises as fs } from "fs"; import http from "http"; import url from "url"; import { Command } from "commander"; import express from "express"; -import { getSharedDepsForLangConfig, readLangConfig } from "../lib/yaml.js"; -import { getLocalImageLabel } from "./docker-util.js"; -import { hashDockerfile } from "./hash-dockerfile.js"; -import { getDebHash, runCommand } from "./util.js"; +import { runCommand } from "./util.js"; // Get a Node.js http server object that will allow the Docker // build to fetch files from outside the container, without them @@ -27,44 +22,6 @@ async function main() { program.option("--debug", "interactive debugging"); program.parse(process.argv); const { lang, debug } = program.opts(); - const sharedDeps = await getSharedDepsForLangConfig( - await readLangConfig(lang) - ); - const installContents = await fs.readFile( - `build/lang/${lang}/install.bash`, - "utf-8" - ); - const sharedInstallContents = await Promise.all( - sharedDeps.map(async (name) => - fs.readFile(`build/shared/${name}/install.bash`) - ) - ); - const allInstallContents = [].concat.apply( - [installContents], - sharedInstallContents - ); - const hash = await hashDockerfile( - "lang", - { - "riju:base": await getLocalImageLabel("riju:base", "riju.image-hash"), - }, - { - salt: { - langHash: await getDebHash(`build/lang/${lang}/riju-lang-${lang}.deb`), - sharedHashes: ( - await Promise.all( - sharedDeps.map( - async (name) => - await getDebHash(`build/shared/${name}/riju-shared-${name}.deb`) - ) - ) - ).sort(), - installHash: allInstallContents - .map((c) => crypto.createHash("sha1").update(c).digest("hex")) - .join(""), - }, - } - ); const server = getServer(); await new Promise((resolve) => server.listen(8487, "localhost", resolve)); try { @@ -76,11 +33,11 @@ async function main() { await runCommand( `docker build . -f docker/lang/Dockerfile ` + `--build-arg LANG=${lang} -t riju:lang-${lang} ` + - `--network host --no-cache --label riju.image-hash=${hash}` + `--network host --no-cache` ); } } finally { - await server.close(); + await new Promise((resolve) => server.close(resolve)); } process.exit(0); } diff --git a/tools/generate-build-script.js b/tools/generate-build-script.js index 329177a..cd7a1d5 100644 --- a/tools/generate-build-script.js +++ b/tools/generate-build-script.js @@ -316,8 +316,7 @@ Maintainer: Radian LLC Description: The ${name} ${ isShared ? "shared dependency" : "language" } packaged for Riju -Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g') -Riju-Script-Hash: \$((cat "\$0"; echo "\${RIJU_IMAGE_HASH}") | sha1sum - | awk '{ print \$1 }')`; +Depends: \$(IFS=,; echo "\${depends[*]}" | sed -E 's/^[ ,]*|[ ,]*$| *(, *)+/},{/g' | sed -E 's/ *(\\| *)+/}\\|{/g'${stripDependsFilter} | tr -d '{}' | sed -E 's/^[,|]+|[,|]+$//g' | sed -E 's/[,|]*,[,|]*/,/g' | sed -E 's/\\|+/|/g')`; parts.push(`\ install -d "\${pkg}/DEBIAN" cat < "\${pkg}/DEBIAN/control" From bcc144ea08bd630cb9458a3237e6591233854ed6 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 09:50:49 -0700 Subject: [PATCH 087/123] Migrate to LTS Ubuntu base images --- docker/base/Dockerfile | 3 ++- docker/packaging/Dockerfile | 3 ++- docker/runtime/Dockerfile | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 60e18e5..572a50e 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -1,4 +1,5 @@ -FROM riju:ubuntu +# EOL: April 2027 +FROM ubuntu:22.04 COPY docker/base/install.bash /tmp/ RUN /tmp/install.bash diff --git a/docker/packaging/Dockerfile b/docker/packaging/Dockerfile index ba528b1..96e0aa9 100644 --- a/docker/packaging/Dockerfile +++ b/docker/packaging/Dockerfile @@ -1,4 +1,5 @@ -FROM riju:ubuntu +# EOL: April 2027 +FROM ubuntu:22.04 COPY docker/packaging/install.bash /tmp/ RUN /tmp/install.bash diff --git a/docker/runtime/Dockerfile b/docker/runtime/Dockerfile index e659167..b8f7373 100644 --- a/docker/runtime/Dockerfile +++ b/docker/runtime/Dockerfile @@ -1,4 +1,5 @@ -FROM riju:ubuntu +# EOL: April 2027 +FROM ubuntu:22.04 COPY docker/runtime/install.bash /tmp/ RUN /tmp/install.bash From 60058f4acf14186af8c819ef0aa3337955fd44fc Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 09:51:06 -0700 Subject: [PATCH 088/123] Microsoft Python LS => Pyright --- langs/python.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/langs/python.yaml b/langs/python.yaml index a024b06..4f2989c 100644 --- a/langs/python.yaml +++ b/langs/python.yaml @@ -41,15 +41,8 @@ install: - python3 - python3-pip - black - manual: | - install -d "${pkg}/opt/mspyls" - install -d "${pkg}/usr/local/bin" - - url="$(curl -fsSL "https://pvsc.blob.core.windows.net/python-language-server-stable?restype=container&comp=list&prefix=Python-Language-Server-linux-x64" | grep -Eo 'https://[^<]+\.nupkg' | tail -n1)" - wget "${url}" - unzip -d "${pkg}/opt/mspyls" Python-Language-Server-linux-x64.*.nupkg - chmod +x "${pkg}/opt/mspyls/Microsoft.Python.LanguageServer" - ln -s "/opt/mspyls/Microsoft.Python.LanguageServer" "${pkg}/usr/local/bin/Microsoft.Python.LanguageServer" + npm: + - pyright repl: | python3 -u @@ -83,11 +76,7 @@ pkg: lsp: start: | - Microsoft.Python.LanguageServer - init: - interpreter: - properties: - InterpreterPath: /usr/bin/python3 + pyright-langserver --stdio code: "import func" item: "functools" From 38d581ecdddced06aad50dc8c139458ada6267aa Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 11:45:13 -0700 Subject: [PATCH 089/123] Build app using Ubuntu 20.04 --- docker/app/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index c9d8ebc..7e919a0 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,5 +1,5 @@ -# temporary hack -FROM ubuntu:20.04 AS build +# EOL: April 2027 +FROM ubuntu:22.04 AS build COPY docker/app/install-build.bash /tmp/ RUN /tmp/install-build.bash From c4356a2e05f0eb18f41ddcde0febf1986600e840 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 14:20:44 -0700 Subject: [PATCH 090/123] Pass raw deploy config into app container --- supervisor/src/main.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 01f4ac1..bac34f0 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -282,8 +282,9 @@ func (sv *supervisor) reload() error { }); err != nil { return err } + deployCfgBytes := buf.Bytes() deployCfg := deploymentConfig{} - if err := json.Unmarshal(buf.Bytes(), &deployCfg); err != nil { + if err := json.Unmarshal(deployCfgBytes, &deployCfg); err != nil { return err } sv.status("listing locally available images") @@ -335,12 +336,8 @@ func (sv *supervisor) reload() error { } } } - deployCfgStr, err := json.Marshal(&deployCfg) - if err != nil { - return err - } h := sha1.New() - h.Write([]byte(deployCfgStr)) + h.Write(deployCfgBytes) deployCfgHash := fmt.Sprintf("%x", h.Sum(nil)) if deployCfgHash == sv.deployConfigHash { sv.status(fmt.Sprintf("config hash remains at %s", deployCfgHash)) @@ -384,7 +381,7 @@ func (sv *supervisor) reload() error { ) dockerRun.Stdout = os.Stdout dockerRun.Stderr = os.Stderr - dockerRun.Env = append(os.Environ(), fmt.Sprintf("RIJU_DEPLOY_CONFIG=%s", deployCfgStr)) + dockerRun.Env = append(os.Environ(), fmt.Sprintf("RIJU_DEPLOY_CONFIG=%s", deployCfgBytes)) if err := dockerRun.Run(); err != nil { return err } From 791ebf08e7371a075cca2d7a2e1b7bfa6b1bd548 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 14:26:45 -0700 Subject: [PATCH 091/123] Upgrade AMI to 22.04 --- packer/web.pkr.hcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index afd6b66..b823d46 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -60,7 +60,8 @@ variable "supervisor_access_token" { data "amazon-ami" "ubuntu" { filters = { - name = "ubuntu/images/hvm-ssd/ubuntu-*-21.04-amd64-server-*" + // EOL: April 2027 + name = "ubuntu/images/hvm-ssd/ubuntu-*-22.04-amd64-server-*" root-device-type = "ebs" virtualization-type = "hvm" } From a703944915f9ea3ae6c777f9378aa2fc20182e15 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 16:35:11 -0700 Subject: [PATCH 092/123] Add ability to pull Docker images via setuid bin --- system/src/riju-system-privileged.c | 42 ++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index b3bf6a6..524d30f 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -30,6 +30,7 @@ void init() { sentinel_bash[sentinel_bash_len - 1] = '\0'; } void die_with_usage() { die("usage:\n" + " riju-system-privileged pull REPO:TAG\n" " riju-system-privileged session UUID LANG [IMAGE-HASH]\n" " riju-system-privileged exec UUID CMDLINE...\n" " riju-system-privileged pty UUID CMDLINE...\n" @@ -117,6 +118,17 @@ char *parseImageHash(char *imageHash) return imageHash; } +char *parseImage(char *image) +{ + if (strnlen(image, 1025) > 1024) + die("illegal image name"); + for (char *ptr = image; *ptr; ++ptr) + if (!((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= '0' && *ptr <= '9') || + *ptr == ':' || *ptr == '/' || *ptr == '.' || *ptr == '-')) + die("illegal image name"); + return image; +} + char *timeout_msg; void sigalrm_die(int signum) @@ -132,7 +144,16 @@ void sigalrm_kill_parent(int signum) exit(EXIT_FAILURE); } -void session(char *uuid, char *lang, char *imageHash) +void cmd_pull(char *image) +{ + char *argv[] = { + "docker", "pull", "--", image, NULL, + }; + execvp(argv[0], argv); + die("execvp failed"); +} + +void cmd_session(char *uuid, char *lang, char *imageHash) { if (setvbuf(stdout, NULL, _IONBF, 0) != 0) die("setvbuf failed"); @@ -295,7 +316,7 @@ void session(char *uuid, char *lang, char *imageHash) } } -void exec(char *uuid, int argc, char **cmdline, bool pty) +void cmd_exec(char *uuid, int argc, char **cmdline, bool pty) { if (setvbuf(stdout, NULL, _IONBF, 0) != 0) die("setvbuf failed"); @@ -463,7 +484,7 @@ void exec(char *uuid, int argc, char **cmdline, bool pty) } } -void teardown(char *uuid) +void cmd_teardown(char *uuid) { if (setuid(0) != 0) die("setuid failed"); @@ -491,31 +512,38 @@ int main(int argc, char **argv) die("seteuid failed"); if (argc < 2) die_with_usage(); + if (!strcmp(argv[1], "pull")) { + if (argc != 3) + die_with_usage(); + char *image = parseImage(argv[2]); + cmd_pull(image); + return 0; + } if (!strcmp(argv[1], "session")) { if (argc < 4 || argc > 5) die_with_usage(); char *uuid = parseUUID(argv[2]); char *lang = parseLang(argv[3]); char *imageHash = argc == 5 ? parseImageHash(argv[4]) : NULL; - session(uuid, lang, imageHash); + cmd_session(uuid, lang, imageHash); return 0; } if (!strcmp(argv[1], "exec")) { if (argc < 4) die_with_usage(); - exec(parseUUID(argv[2]), argc - 3, &argv[3], false); + cmd_exec(parseUUID(argv[2]), argc - 3, &argv[3], false); return 0; } if (!strcmp(argv[1], "pty")) { if (argc < 4) die_with_usage(); - exec(parseUUID(argv[2]), argc - 3, &argv[3], true); + cmd_exec(parseUUID(argv[2]), argc - 3, &argv[3], true); return 0; } if (!strcmp(argv[1], "teardown")) { if (argc < 2) die_with_usage(); - teardown(argc >= 3 ? parseUUID(argv[2]) : NULL); + cmd_teardown(argc >= 3 ? parseUUID(argv[2]) : NULL); return 0; } die_with_usage(); From 31b01855053c515d5dffa866f5098e7207bb2ae5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 10 Oct 2022 21:51:25 -0700 Subject: [PATCH 093/123] More work on pull and list subcmds --- backend/util.js | 8 ++ system/src/riju-system-privileged.c | 112 ++++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 14 deletions(-) diff --git a/backend/util.js b/backend/util.js index 4e5d7b4..9113ab2 100644 --- a/backend/util.js +++ b/backend/util.js @@ -86,6 +86,14 @@ export async function run(args, log, options) { }); } +export function privilegedList() { + return [rijuSystemPrivileged, "list"]; +} + +export function privilegedPull({ repo, tag }) { + return [rijuSystemPrivileged, "pull", repo, tag]; +} + export function privilegedSession({ uuid, lang }) { const cmdline = [rijuSystemPrivileged, "session", uuid, lang]; if (imageHashes[lang]) { diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index 524d30f..80544c1 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -30,7 +30,8 @@ void init() { sentinel_bash[sentinel_bash_len - 1] = '\0'; } void die_with_usage() { die("usage:\n" - " riju-system-privileged pull REPO:TAG\n" + " riju-system-privileged list\n" + " riju-system-privileged pull REPO TAG\n" " riju-system-privileged session UUID LANG [IMAGE-HASH]\n" " riju-system-privileged exec UUID CMDLINE...\n" " riju-system-privileged pty UUID CMDLINE...\n" @@ -118,15 +119,26 @@ char *parseImageHash(char *imageHash) return imageHash; } -char *parseImage(char *image) +char *parseRepo(char *repo) { - if (strnlen(image, 1025) > 1024) - die("illegal image name"); - for (char *ptr = image; *ptr; ++ptr) + if (strnlen(repo, 501) > 500) + die("illegal repo name"); + for (char *ptr = repo; *ptr; ++ptr) if (!((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= '0' && *ptr <= '9') || - *ptr == ':' || *ptr == '/' || *ptr == '.' || *ptr == '-')) - die("illegal image name"); - return image; + *ptr == '/' || *ptr == '.' || *ptr == '-' || *ptr == '_')) + die("illegal repo name"); + return repo; +} + +char *parseTag(char *tag) +{ + if (strnlen(tag, 501) > 500) + die("illegal tag name"); + for (char *ptr = tag; *ptr; ++ptr) + if (!((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= '0' && *ptr <= '9') || + *ptr == '.' || *ptr == '-' || *ptr == '_')) + die("illegal tag name"); + return tag; } char *timeout_msg; @@ -144,10 +156,75 @@ void sigalrm_kill_parent(int signum) exit(EXIT_FAILURE); } -void cmd_pull(char *image) +void cmd_list() { + // This command prints a bunch of empty lines because there is no + // way to filter to a desired set of images. Caller is expected to + // remove empty lines because it's easier in JS than C. char *argv[] = { - "docker", "pull", "--", image, NULL, + "docker", + "image", + "ls", + "--format", + "{{ if eq .Repository \"riju\" }}{{ .Tag }}{{ end }}", + NULL, + }; + execvp(argv[0], argv); + die("execvp failed"); +} + +void cmd_pull(char *repo, char *tag) +{ + char *localImage, *remoteImage; + if (asprintf(&remoteImage, "%s:%s", repo, tag) < 0) + die("asprintf failed"); + if (asprintf(&localImage, "riju:%s", tag) < 0) + die("asprintf failed"); + pid_t orig_ppid = getpid(); + pid_t pid = fork(); + if (pid < 0) + die("fork failed"); + else if (pid == 0) { + if (freopen("/dev/null", "w", stdout) == NULL) + die("freopen failed"); + if (prctl(PR_SET_PDEATHSIG, SIGTERM) < 0) + die("prctl failed"); + if (getppid() != orig_ppid) + exit(EXIT_FAILURE); + char *argv[] = { + "docker", "inspect", "--", localImage, NULL, + }; + execvp(argv[0], argv); + die("execvp failed"); + } + siginfo_t info; + if (waitid(P_PID, pid, &info, WEXITED) < 0) + die("waitid failed"); + if (info.si_status == 0) { + // Image exists already, no need to pull. It is only appropriate + // to use cmd_pull with immutable images. + return; + } + orig_ppid = getpid(); + pid = fork(); + if (pid < 0) + die("fork failed"); + else if (pid == 0) { + if (prctl(PR_SET_PDEATHSIG, SIGTERM) < 0) + die("prctl failed"); + if (getppid() != orig_ppid) + exit(EXIT_FAILURE); + char *argv[] = { + "docker", "pull", "--", remoteImage, NULL, + }; + execvp(argv[0], argv); + } + if (waitid(P_PID, pid, &info, WEXITED) < 0) + die("waitid failed"); + if (info.si_status != 0) + die("child process failed"); + char *argv[] = { + "docker", "tag", "--", remoteImage, localImage, }; execvp(argv[0], argv); die("execvp failed"); @@ -512,11 +589,18 @@ int main(int argc, char **argv) die("seteuid failed"); if (argc < 2) die_with_usage(); - if (!strcmp(argv[1], "pull")) { - if (argc != 3) + if (!strcmp(argv[1], "list")) { + if (argc != 2) die_with_usage(); - char *image = parseImage(argv[2]); - cmd_pull(image); + cmd_list(); + return 0; + } + if (!strcmp(argv[1], "pull")) { + if (argc != 4) + die_with_usage(); + char *repo = parseRepo(argv[2]); + char *tag = parseTag(argv[3]); + cmd_pull(repo, tag); return 0; } if (!strcmp(argv[1], "session")) { From 59a92b2ca4809bf0a4391a797b6ba4a3f0a84193 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 19 Dec 2022 15:14:23 -0700 Subject: [PATCH 094/123] Start migrating to Kubernetes in earnest --- .gitignore | 5 +- env.yaml.bash | 20 + k8s/cluster.yaml | 105 ++ k8s/metallb-config.in.yaml | 9 + k8s/metallb-crds.yaml | 1108 +++++++++++++++++ k8s/metallb-rbac.yaml | 352 ++++++ k8s/metallb.yaml | 326 +++++ k8s/namespaces.yaml | 17 + k8s/provisioning.md | 25 + k8s/riju.yaml | 124 ++ k8s/secrets.in.yaml | 17 + k8s/traefik-crds.yaml | 2288 ++++++++++++++++++++++++++++++++++++ k8s/traefik-rbac.yaml | 72 ++ k8s/traefik.yaml | 145 +++ tools/template/Makefile | 5 + tools/template/go.mod | 8 + tools/template/go.sum | 39 + tools/template/template | 7 + tools/template/template.go | 88 ++ 19 files changed, 4758 insertions(+), 2 deletions(-) create mode 100755 env.yaml.bash create mode 100644 k8s/cluster.yaml create mode 100644 k8s/metallb-config.in.yaml create mode 100644 k8s/metallb-crds.yaml create mode 100644 k8s/metallb-rbac.yaml create mode 100644 k8s/metallb.yaml create mode 100644 k8s/namespaces.yaml create mode 100644 k8s/provisioning.md create mode 100644 k8s/riju.yaml create mode 100644 k8s/secrets.in.yaml create mode 100644 k8s/traefik-crds.yaml create mode 100644 k8s/traefik-rbac.yaml create mode 100644 k8s/traefik.yaml create mode 100644 tools/template/Makefile create mode 100644 tools/template/go.mod create mode 100644 tools/template/go.sum create mode 100755 tools/template/template create mode 100644 tools/template/template.go diff --git a/.gitignore b/.gitignore index 8aeedf7..207808b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,13 @@ *.log *.out +*.out.* *.pem .env .lsp-repl-history .terraform +bin build +env.yaml node_modules out sentinel.h -financials/????-??/* -!financials/????-??/breakdown.txt diff --git a/env.yaml.bash b/env.yaml.bash new file mode 100755 index 0000000..4720096 --- /dev/null +++ b/env.yaml.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "$(dirname "$0")" + +registry_password="$(pwgen -s 20 1)" + +cat < /etc/k0s/k0s.yaml +``` + +Edit to have this config: + +```yaml +spec: + extensions: + storage: + type: openebs_local_storage +``` + +```bash +sudo k0s install controller --single +sudo k0s start +``` + +Go to client machine: + +```bash +ssh riju-k8s sudo -S k0s kubeconfig admin > ~/.kube/config +``` diff --git a/k8s/riju.yaml b/k8s/riju.yaml new file mode 100644 index 0000000..dc25937 --- /dev/null +++ b/k8s/riju.yaml @@ -0,0 +1,124 @@ +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + namespace: riju + name: docker-registry +spec: + replicas: 1 + serviceName: docker-registry + selector: + matchLabels: + app: docker-registry + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 128Gi + storageClassName: openebs-hostpath + template: + metadata: + labels: + app: docker-registry + spec: + volumes: + - name: auth + secret: + secretName: registry-auth + containers: + - name: registry + image: "registry:2" + env: + - name: REGISTRY_AUTH + value: htpasswd + - name: REGISTRY_AUTH_HTPASSWD_REALM + value: "Registry Realm" + - name: REGISTRY_AUTH_HTPASSWD_PATH + value: /var/run/registry/auth/htpasswd + ports: + - name: api + containerPort: 5000 + volumeMounts: + - name: auth + mountPath: /var/run/registry/auth + - name: data + mountPath: /var/lib/registry + +--- +kind: Service +apiVersion: v1 +metadata: + namespace: riju + name: docker-registry +spec: + type: NodePort + selector: + app: docker-registry + ports: + - name: api + port: 80 + nodePort: 30999 + targetPort: 5000 + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: riju + name: riju-server +spec: + replicas: 1 + selector: + matchLabels: + app: riju-server + template: + metadata: + labels: + app: riju-server + spec: + volumes: + - name: cache + hostPath: + path: /var/cache/riju + - name: docker + hostPath: + path: /var/run/docker.sock + containers: + - name: server + image: "localhost:30999/riju:app" + resources: + requests: + cpu: "1000m" + memory: "6Gi" + limits: + cpu: "1000m" + memory: "6Gi" + ports: + - name: http + containerPort: 6119 + volumeMounts: + - name: cache + mountPath: /var/cache/riju + - name: docker + mountPath: /var/run/docker.sock + readOnly: true + +--- +kind: Service +apiVersion: v1 +metadata: + namespace: riju + name: riju-server +spec: + selector: + app: riju-server + ports: + - name: http + port: 80 + targetPort: 6119 + +--- +kind: Ingress diff --git a/k8s/secrets.in.yaml b/k8s/secrets.in.yaml new file mode 100644 index 0000000..9ca53a3 --- /dev/null +++ b/k8s/secrets.in.yaml @@ -0,0 +1,17 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: metallb + name: metallb-memberlist +data: + secretkey: "{{ .metallb.secretkey | b64enc }}" + +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: riju + name: registry-auth +data: + htpasswd: "{{ .registry.htpasswd | println | b64enc }}" diff --git a/k8s/traefik-crds.yaml b/k8s/traefik-crds.yaml new file mode 100644 index 0000000..a4b290e --- /dev/null +++ b/k8s/traefik-crds.yaml @@ -0,0 +1,2288 @@ +# https://github.com/traefik/traefik/blob/v2.9/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: ingressroutes.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: IngressRoute + listKind: IngressRouteList + plural: ingressroutes + singular: ingressroute + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IngressRoute is the CRD implementation of a Traefik HTTP Router. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IngressRouteSpec defines the desired state of IngressRoute. + properties: + entryPoints: + description: 'EntryPoints defines the list of entry point names to + bind to. Entry points have to be configured in the static configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/entrypoints/ + Default: all.' + items: + type: string + type: array + routes: + description: Routes defines the list of routes. + items: + description: Route holds the HTTP route configuration. + properties: + kind: + description: Kind defines the kind of the route. Rule is the + only supported kind. + enum: + - Rule + type: string + match: + description: 'Match defines the router''s rule. More info: https://doc.traefik.io/traefik/v2.9/routing/routers/#rule' + type: string + middlewares: + description: 'Middlewares defines the list of references to + Middleware resources. More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#kind-middleware' + items: + description: MiddlewareRef is a reference to a Middleware + resource. + properties: + name: + description: Name defines the name of the referenced Middleware + resource. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Middleware resource. + type: string + required: + - name + type: object + type: array + priority: + description: 'Priority defines the router''s priority. More + info: https://doc.traefik.io/traefik/v2.9/routing/routers/#priority' + type: integer + services: + description: Services defines the list of Service. It can contain + any combination of TraefikService and/or reference to a Kubernetes + Service. + items: + description: Service defines an upstream HTTP service to proxy + traffic to. + properties: + kind: + description: Kind defines the kind of the Service. + enum: + - Service + - TraefikService + type: string + name: + description: Name defines the name of the referenced Kubernetes + Service or TraefikService. The differentiation between + the two is specified in the Kind field. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service or TraefikService. + type: string + passHostHeader: + description: PassHostHeader defines whether the client + Host header is forwarded to the upstream Kubernetes + Service. By default, passHostHeader is true. + type: boolean + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. + This can be a reference to a named port. + x-kubernetes-int-or-string: true + responseForwarding: + description: ResponseForwarding defines how Traefik forwards + the response from the upstream Kubernetes Service to + the client. + properties: + flushInterval: + description: 'FlushInterval defines the interval, + in milliseconds, in between flushes to the client + while copying the response body. A negative value + means to flush immediately after each write to the + client. This configuration is ignored when ReverseProxy + recognizes a response as a streaming response; for + such responses, writes are flushed to the client + immediately. Default: 100ms' + type: string + type: object + scheme: + description: Scheme defines the scheme to use for the + request to the upstream Kubernetes Service. It defaults + to https when Kubernetes Service port is 443, http otherwise. + type: string + serversTransport: + description: ServersTransport defines the name of ServersTransport + resource to use. It allows to configure the transport + between Traefik and your servers. Can only be used on + a Kubernetes Service. + type: string + sticky: + description: 'Sticky defines the sticky sessions configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/services/#sticky-sessions' + properties: + cookie: + description: Cookie defines the sticky cookie configuration. + properties: + httpOnly: + description: HTTPOnly defines whether the cookie + can be accessed by client-side APIs, such as + JavaScript. + type: boolean + name: + description: Name defines the Cookie name. + type: string + sameSite: + description: 'SameSite defines the same site policy. + More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite' + type: string + secure: + description: Secure defines whether the cookie + can only be transmitted over an encrypted connection + (i.e. HTTPS). + type: boolean + type: object + type: object + strategy: + description: Strategy defines the load balancing strategy + between the servers. RoundRobin is the only supported + value at the moment. + type: string + weight: + description: Weight defines the weight and should only + be specified when Name references a TraefikService object + (and to be precise, one that embeds a Weighted Round + Robin). + type: integer + required: + - name + type: object + type: array + required: + - kind + - match + type: object + type: array + tls: + description: 'TLS defines the TLS configuration. More info: https://doc.traefik.io/traefik/v2.9/routing/routers/#tls' + properties: + certResolver: + description: 'CertResolver defines the name of the certificate + resolver to use. Cert resolvers have to be configured in the + static configuration. More info: https://doc.traefik.io/traefik/v2.9/https/acme/#certificate-resolvers' + type: string + domains: + description: 'Domains defines the list of domains that will be + used to issue certificates. More info: https://doc.traefik.io/traefik/v2.9/routing/routers/#domains' + items: + description: Domain holds a domain name with SANs. + properties: + main: + description: Main defines the main domain name. + type: string + sans: + description: SANs defines the subject alternative domain + names. + items: + type: string + type: array + type: object + type: array + options: + description: 'Options defines the reference to a TLSOption, that + specifies the parameters of the TLS connection. If not defined, + the `default` TLSOption is used. More info: https://doc.traefik.io/traefik/v2.9/https/tls/#tls-options' + properties: + name: + description: 'Name defines the name of the referenced TLSOption. + More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#kind-tlsoption' + type: string + namespace: + description: 'Namespace defines the namespace of the referenced + TLSOption. More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#kind-tlsoption' + type: string + required: + - name + type: object + secretName: + description: SecretName is the name of the referenced Kubernetes + Secret to specify the certificate details. + type: string + store: + description: Store defines the reference to the TLSStore, that + will be used to store certificates. Please note that only `default` + TLSStore can be used. + properties: + name: + description: 'Name defines the name of the referenced TLSStore. + More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#kind-tlsstore' + type: string + namespace: + description: 'Namespace defines the namespace of the referenced + TLSStore. More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#kind-tlsstore' + type: string + required: + - name + type: object + type: object + required: + - routes + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: ingressroutetcps.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: IngressRouteTCP + listKind: IngressRouteTCPList + plural: ingressroutetcps + singular: ingressroutetcp + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IngressRouteTCP is the CRD implementation of a Traefik TCP Router. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IngressRouteTCPSpec defines the desired state of IngressRouteTCP. + properties: + entryPoints: + description: 'EntryPoints defines the list of entry point names to + bind to. Entry points have to be configured in the static configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/entrypoints/ + Default: all.' + items: + type: string + type: array + routes: + description: Routes defines the list of routes. + items: + description: RouteTCP holds the TCP route configuration. + properties: + match: + description: 'Match defines the router''s rule. More info: https://doc.traefik.io/traefik/v2.9/routing/routers/#rule_1' + type: string + middlewares: + description: Middlewares defines the list of references to MiddlewareTCP + resources. + items: + description: ObjectReference is a generic reference to a Traefik + resource. + properties: + name: + description: Name defines the name of the referenced Traefik + resource. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Traefik resource. + type: string + required: + - name + type: object + type: array + priority: + description: 'Priority defines the router''s priority. More + info: https://doc.traefik.io/traefik/v2.9/routing/routers/#priority_1' + type: integer + services: + description: Services defines the list of TCP services. + items: + description: ServiceTCP defines an upstream TCP service to + proxy traffic to. + properties: + name: + description: Name defines the name of the referenced Kubernetes + Service. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service. + type: string + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. + This can be a reference to a named port. + x-kubernetes-int-or-string: true + proxyProtocol: + description: 'ProxyProtocol defines the PROXY protocol + configuration. More info: https://doc.traefik.io/traefik/v2.9/routing/services/#proxy-protocol' + properties: + version: + description: Version defines the PROXY Protocol version + to use. + type: integer + type: object + terminationDelay: + description: TerminationDelay defines the deadline that + the proxy sets, after one of its connected peers indicates + it has closed the writing capability of its connection, + to close the reading capability as well, hence fully + terminating the connection. It is a duration in milliseconds, + defaulting to 100. A negative value means an infinite + deadline (i.e. the reading capability is never closed). + type: integer + weight: + description: Weight defines the weight used when balancing + requests between multiple Kubernetes Service. + type: integer + required: + - name + - port + type: object + type: array + required: + - match + type: object + type: array + tls: + description: 'TLS defines the TLS configuration on a layer 4 / TCP + Route. More info: https://doc.traefik.io/traefik/v2.9/routing/routers/#tls_1' + properties: + certResolver: + description: 'CertResolver defines the name of the certificate + resolver to use. Cert resolvers have to be configured in the + static configuration. More info: https://doc.traefik.io/traefik/v2.9/https/acme/#certificate-resolvers' + type: string + domains: + description: 'Domains defines the list of domains that will be + used to issue certificates. More info: https://doc.traefik.io/traefik/v2.9/routing/routers/#domains' + items: + description: Domain holds a domain name with SANs. + properties: + main: + description: Main defines the main domain name. + type: string + sans: + description: SANs defines the subject alternative domain + names. + items: + type: string + type: array + type: object + type: array + options: + description: 'Options defines the reference to a TLSOption, that + specifies the parameters of the TLS connection. If not defined, + the `default` TLSOption is used. More info: https://doc.traefik.io/traefik/v2.9/https/tls/#tls-options' + properties: + name: + description: Name defines the name of the referenced Traefik + resource. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Traefik resource. + type: string + required: + - name + type: object + passthrough: + description: Passthrough defines whether a TLS router will terminate + the TLS connection. + type: boolean + secretName: + description: SecretName is the name of the referenced Kubernetes + Secret to specify the certificate details. + type: string + store: + description: Store defines the reference to the TLSStore, that + will be used to store certificates. Please note that only `default` + TLSStore can be used. + properties: + name: + description: Name defines the name of the referenced Traefik + resource. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Traefik resource. + type: string + required: + - name + type: object + type: object + required: + - routes + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: ingressrouteudps.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: IngressRouteUDP + listKind: IngressRouteUDPList + plural: ingressrouteudps + singular: ingressrouteudp + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IngressRouteUDP is a CRD implementation of a Traefik UDP Router. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IngressRouteUDPSpec defines the desired state of a IngressRouteUDP. + properties: + entryPoints: + description: 'EntryPoints defines the list of entry point names to + bind to. Entry points have to be configured in the static configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/entrypoints/ + Default: all.' + items: + type: string + type: array + routes: + description: Routes defines the list of routes. + items: + description: RouteUDP holds the UDP route configuration. + properties: + services: + description: Services defines the list of UDP services. + items: + description: ServiceUDP defines an upstream UDP service to + proxy traffic to. + properties: + name: + description: Name defines the name of the referenced Kubernetes + Service. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service. + type: string + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. + This can be a reference to a named port. + x-kubernetes-int-or-string: true + weight: + description: Weight defines the weight used when balancing + requests between multiple Kubernetes Service. + type: integer + required: + - name + - port + type: object + type: array + type: object + type: array + required: + - routes + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: middlewares.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: Middleware + listKind: MiddlewareList + plural: middlewares + singular: middleware + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'Middleware is the CRD implementation of a Traefik Middleware. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/overview/' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MiddlewareSpec defines the desired state of a Middleware. + properties: + addPrefix: + description: 'AddPrefix holds the add prefix middleware configuration. + This middleware updates the path of a request before forwarding + it. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/addprefix/' + properties: + prefix: + description: Prefix is the string to add before the current path + in the requested URL. It should include a leading slash (/). + type: string + type: object + basicAuth: + description: 'BasicAuth holds the basic auth middleware configuration. + This middleware restricts access to your services to known users. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/basicauth/' + properties: + headerField: + description: 'HeaderField defines a header field to store the + authenticated user. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/basicauth/#headerfield' + type: string + realm: + description: 'Realm allows the protected resources on a server + to be partitioned into a set of protection spaces, each with + its own authentication scheme. Default: traefik.' + type: string + removeHeader: + description: 'RemoveHeader sets the removeHeader option to true + to remove the authorization header before forwarding the request + to your service. Default: false.' + type: boolean + secret: + description: Secret is the name of the referenced Kubernetes Secret + containing user credentials. + type: string + type: object + buffering: + description: 'Buffering holds the buffering middleware configuration. + This middleware retries or limits the size of requests that can + be forwarded to backends. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/buffering/#maxrequestbodybytes' + properties: + maxRequestBodyBytes: + description: 'MaxRequestBodyBytes defines the maximum allowed + body size for the request (in bytes). If the request exceeds + the allowed size, it is not forwarded to the service, and the + client gets a 413 (Request Entity Too Large) response. Default: + 0 (no maximum).' + format: int64 + type: integer + maxResponseBodyBytes: + description: 'MaxResponseBodyBytes defines the maximum allowed + response size from the service (in bytes). If the response exceeds + the allowed size, it is not forwarded to the client. The client + gets a 500 (Internal Server Error) response instead. Default: + 0 (no maximum).' + format: int64 + type: integer + memRequestBodyBytes: + description: 'MemRequestBodyBytes defines the threshold (in bytes) + from which the request will be buffered on disk instead of in + memory. Default: 1048576 (1Mi).' + format: int64 + type: integer + memResponseBodyBytes: + description: 'MemResponseBodyBytes defines the threshold (in bytes) + from which the response will be buffered on disk instead of + in memory. Default: 1048576 (1Mi).' + format: int64 + type: integer + retryExpression: + description: 'RetryExpression defines the retry conditions. It + is a logical combination of functions with operators AND (&&) + and OR (||). More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/buffering/#retryexpression' + type: string + type: object + chain: + description: 'Chain holds the configuration of the chain middleware. + This middleware enables to define reusable combinations of other + pieces of middleware. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/chain/' + properties: + middlewares: + description: Middlewares is the list of MiddlewareRef which composes + the chain. + items: + description: MiddlewareRef is a reference to a Middleware resource. + properties: + name: + description: Name defines the name of the referenced Middleware + resource. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Middleware resource. + type: string + required: + - name + type: object + type: array + type: object + circuitBreaker: + description: CircuitBreaker holds the circuit breaker configuration. + properties: + checkPeriod: + anyOf: + - type: integer + - type: string + description: CheckPeriod is the interval between successive checks + of the circuit breaker condition (when in standby state). + x-kubernetes-int-or-string: true + expression: + description: Expression is the condition that triggers the tripped + state. + type: string + fallbackDuration: + anyOf: + - type: integer + - type: string + description: FallbackDuration is the duration for which the circuit + breaker will wait before trying to recover (from a tripped state). + x-kubernetes-int-or-string: true + recoveryDuration: + anyOf: + - type: integer + - type: string + description: RecoveryDuration is the duration for which the circuit + breaker will try to recover (as soon as it is in recovering + state). + x-kubernetes-int-or-string: true + type: object + compress: + description: 'Compress holds the compress middleware configuration. + This middleware compresses responses before sending them to the + client, using gzip compression. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/compress/' + properties: + excludedContentTypes: + description: ExcludedContentTypes defines the list of content + types to compare the Content-Type header of the incoming requests + and responses before compressing. + items: + type: string + type: array + minResponseBodyBytes: + description: 'MinResponseBodyBytes defines the minimum amount + of bytes a response body must have to be compressed. Default: + 1024.' + type: integer + type: object + contentType: + description: ContentType holds the content-type middleware configuration. + This middleware exists to enable the correct behavior until at least + the default one can be changed in a future version. + properties: + autoDetect: + description: AutoDetect specifies whether to let the `Content-Type` + header, if it has not been set by the backend, be automatically + set to a value derived from the contents of the response. As + a proxy, the default behavior should be to leave the header + alone, regardless of what the backend did with it. However, + the historic default was to always auto-detect and set the header + if it was nil, and it is going to be kept that way in order + to support users currently relying on it. + type: boolean + type: object + digestAuth: + description: 'DigestAuth holds the digest auth middleware configuration. + This middleware restricts access to your services to known users. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/digestauth/' + properties: + headerField: + description: 'HeaderField defines a header field to store the + authenticated user. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/basicauth/#headerfield' + type: string + realm: + description: 'Realm allows the protected resources on a server + to be partitioned into a set of protection spaces, each with + its own authentication scheme. Default: traefik.' + type: string + removeHeader: + description: RemoveHeader defines whether to remove the authorization + header before forwarding the request to the backend. + type: boolean + secret: + description: Secret is the name of the referenced Kubernetes Secret + containing user credentials. + type: string + type: object + errors: + description: 'ErrorPage holds the custom error middleware configuration. + This middleware returns a custom page in lieu of the default, according + to configured ranges of HTTP Status codes. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/errorpages/' + properties: + query: + description: Query defines the URL for the error page (hosted + by service). The {status} variable can be used in order to insert + the status code in the URL. + type: string + service: + description: 'Service defines the reference to a Kubernetes Service + that will serve the error page. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/errorpages/#service' + properties: + kind: + description: Kind defines the kind of the Service. + enum: + - Service + - TraefikService + type: string + name: + description: Name defines the name of the referenced Kubernetes + Service or TraefikService. The differentiation between the + two is specified in the Kind field. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service or TraefikService. + type: string + passHostHeader: + description: PassHostHeader defines whether the client Host + header is forwarded to the upstream Kubernetes Service. + By default, passHostHeader is true. + type: boolean + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. + This can be a reference to a named port. + x-kubernetes-int-or-string: true + responseForwarding: + description: ResponseForwarding defines how Traefik forwards + the response from the upstream Kubernetes Service to the + client. + properties: + flushInterval: + description: 'FlushInterval defines the interval, in milliseconds, + in between flushes to the client while copying the response + body. A negative value means to flush immediately after + each write to the client. This configuration is ignored + when ReverseProxy recognizes a response as a streaming + response; for such responses, writes are flushed to + the client immediately. Default: 100ms' + type: string + type: object + scheme: + description: Scheme defines the scheme to use for the request + to the upstream Kubernetes Service. It defaults to https + when Kubernetes Service port is 443, http otherwise. + type: string + serversTransport: + description: ServersTransport defines the name of ServersTransport + resource to use. It allows to configure the transport between + Traefik and your servers. Can only be used on a Kubernetes + Service. + type: string + sticky: + description: 'Sticky defines the sticky sessions configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/services/#sticky-sessions' + properties: + cookie: + description: Cookie defines the sticky cookie configuration. + properties: + httpOnly: + description: HTTPOnly defines whether the cookie can + be accessed by client-side APIs, such as JavaScript. + type: boolean + name: + description: Name defines the Cookie name. + type: string + sameSite: + description: 'SameSite defines the same site policy. + More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite' + type: string + secure: + description: Secure defines whether the cookie can + only be transmitted over an encrypted connection + (i.e. HTTPS). + type: boolean + type: object + type: object + strategy: + description: Strategy defines the load balancing strategy + between the servers. RoundRobin is the only supported value + at the moment. + type: string + weight: + description: Weight defines the weight and should only be + specified when Name references a TraefikService object (and + to be precise, one that embeds a Weighted Round Robin). + type: integer + required: + - name + type: object + status: + description: Status defines which status or range of statuses + should result in an error page. It can be either a status code + as a number (500), as multiple comma-separated numbers (500,502), + as ranges by separating two codes with a dash (500-599), or + a combination of the two (404,418,500-599). + items: + type: string + type: array + type: object + forwardAuth: + description: 'ForwardAuth holds the forward auth middleware configuration. + This middleware delegates the request authentication to a Service. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/forwardauth/' + properties: + address: + description: Address defines the authentication server address. + type: string + authRequestHeaders: + description: AuthRequestHeaders defines the list of the headers + to copy from the request to the authentication server. If not + set or empty then all request headers are passed. + items: + type: string + type: array + authResponseHeaders: + description: AuthResponseHeaders defines the list of headers to + copy from the authentication server response and set on forwarded + request, replacing any existing conflicting headers. + items: + type: string + type: array + authResponseHeadersRegex: + description: 'AuthResponseHeadersRegex defines the regex to match + headers to copy from the authentication server response and + set on forwarded request, after stripping all headers that match + the regex. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/forwardauth/#authresponseheadersregex' + type: string + tls: + description: TLS defines the configuration used to secure the + connection to the authentication server. + properties: + caOptional: + type: boolean + caSecret: + description: CASecret is the name of the referenced Kubernetes + Secret containing the CA to validate the server certificate. + The CA certificate is extracted from key `tls.ca` or `ca.crt`. + type: string + certSecret: + description: CertSecret is the name of the referenced Kubernetes + Secret containing the client certificate. The client certificate + is extracted from the keys `tls.crt` and `tls.key`. + type: string + insecureSkipVerify: + description: InsecureSkipVerify defines whether the server + certificates should be validated. + type: boolean + type: object + trustForwardHeader: + description: 'TrustForwardHeader defines whether to trust (ie: + forward) all X-Forwarded-* headers.' + type: boolean + type: object + headers: + description: 'Headers holds the headers middleware configuration. + This middleware manages the requests and responses headers. More + info: https://doc.traefik.io/traefik/v2.9/middlewares/http/headers/#customrequestheaders' + properties: + accessControlAllowCredentials: + description: AccessControlAllowCredentials defines whether the + request can include user credentials. + type: boolean + accessControlAllowHeaders: + description: AccessControlAllowHeaders defines the Access-Control-Request-Headers + values sent in preflight response. + items: + type: string + type: array + accessControlAllowMethods: + description: AccessControlAllowMethods defines the Access-Control-Request-Method + values sent in preflight response. + items: + type: string + type: array + accessControlAllowOriginList: + description: AccessControlAllowOriginList is a list of allowable + origins. Can also be a wildcard origin "*". + items: + type: string + type: array + accessControlAllowOriginListRegex: + description: AccessControlAllowOriginListRegex is a list of allowable + origins written following the Regular Expression syntax (https://golang.org/pkg/regexp/). + items: + type: string + type: array + accessControlExposeHeaders: + description: AccessControlExposeHeaders defines the Access-Control-Expose-Headers + values sent in preflight response. + items: + type: string + type: array + accessControlMaxAge: + description: AccessControlMaxAge defines the time that a preflight + request may be cached. + format: int64 + type: integer + addVaryHeader: + description: AddVaryHeader defines whether the Vary header is + automatically added/updated when the AccessControlAllowOriginList + is set. + type: boolean + allowedHosts: + description: AllowedHosts defines the fully qualified list of + allowed domain names. + items: + type: string + type: array + browserXssFilter: + description: BrowserXSSFilter defines whether to add the X-XSS-Protection + header with the value 1; mode=block. + type: boolean + contentSecurityPolicy: + description: ContentSecurityPolicy defines the Content-Security-Policy + header value. + type: string + contentTypeNosniff: + description: ContentTypeNosniff defines whether to add the X-Content-Type-Options + header with the nosniff value. + type: boolean + customBrowserXSSValue: + description: CustomBrowserXSSValue defines the X-XSS-Protection + header value. This overrides the BrowserXssFilter option. + type: string + customFrameOptionsValue: + description: CustomFrameOptionsValue defines the X-Frame-Options + header value. This overrides the FrameDeny option. + type: string + customRequestHeaders: + additionalProperties: + type: string + description: CustomRequestHeaders defines the header names and + values to apply to the request. + type: object + customResponseHeaders: + additionalProperties: + type: string + description: CustomResponseHeaders defines the header names and + values to apply to the response. + type: object + featurePolicy: + description: 'Deprecated: use PermissionsPolicy instead.' + type: string + forceSTSHeader: + description: ForceSTSHeader defines whether to add the STS header + even when the connection is HTTP. + type: boolean + frameDeny: + description: FrameDeny defines whether to add the X-Frame-Options + header with the DENY value. + type: boolean + hostsProxyHeaders: + description: HostsProxyHeaders defines the header keys that may + hold a proxied hostname value for the request. + items: + type: string + type: array + isDevelopment: + description: IsDevelopment defines whether to mitigate the unwanted + effects of the AllowedHosts, SSL, and STS options when developing. + Usually testing takes place using HTTP, not HTTPS, and on localhost, + not your production domain. If you would like your development + environment to mimic production with complete Host blocking, + SSL redirects, and STS headers, leave this as false. + type: boolean + permissionsPolicy: + description: PermissionsPolicy defines the Permissions-Policy + header value. This allows sites to control browser features. + type: string + publicKey: + description: PublicKey is the public key that implements HPKP + to prevent MITM attacks with forged certificates. + type: string + referrerPolicy: + description: ReferrerPolicy defines the Referrer-Policy header + value. This allows sites to control whether browsers forward + the Referer header to other sites. + type: string + sslForceHost: + description: 'Deprecated: use RedirectRegex instead.' + type: boolean + sslHost: + description: 'Deprecated: use RedirectRegex instead.' + type: string + sslProxyHeaders: + additionalProperties: + type: string + description: 'SSLProxyHeaders defines the header keys with associated + values that would indicate a valid HTTPS request. It can be + useful when using other proxies (example: "X-Forwarded-Proto": + "https").' + type: object + sslRedirect: + description: 'Deprecated: use EntryPoint redirection or RedirectScheme + instead.' + type: boolean + sslTemporaryRedirect: + description: 'Deprecated: use EntryPoint redirection or RedirectScheme + instead.' + type: boolean + stsIncludeSubdomains: + description: STSIncludeSubdomains defines whether the includeSubDomains + directive is appended to the Strict-Transport-Security header. + type: boolean + stsPreload: + description: STSPreload defines whether the preload flag is appended + to the Strict-Transport-Security header. + type: boolean + stsSeconds: + description: STSSeconds defines the max-age of the Strict-Transport-Security + header. If set to 0, the header is not set. + format: int64 + type: integer + type: object + inFlightReq: + description: 'InFlightReq holds the in-flight request middleware configuration. + This middleware limits the number of requests being processed and + served concurrently. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/inflightreq/' + properties: + amount: + description: Amount defines the maximum amount of allowed simultaneous + in-flight request. The middleware responds with HTTP 429 Too + Many Requests if there are already amount requests in progress + (based on the same sourceCriterion strategy). + format: int64 + type: integer + sourceCriterion: + description: 'SourceCriterion defines what criterion is used to + group requests as originating from a common source. If several + strategies are defined at the same time, an error will be raised. + If none are set, the default is to use the requestHost. More + info: https://doc.traefik.io/traefik/v2.9/middlewares/http/inflightreq/#sourcecriterion' + properties: + ipStrategy: + description: 'IPStrategy holds the IP strategy configuration + used by Traefik to determine the client IP. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/ipwhitelist/#ipstrategy' + properties: + depth: + description: Depth tells Traefik to use the X-Forwarded-For + header and take the IP located at the depth position + (starting from the right). + type: integer + excludedIPs: + description: ExcludedIPs configures Traefik to scan the + X-Forwarded-For header and select the first IP not in + the list. + items: + type: string + type: array + type: object + requestHeaderName: + description: RequestHeaderName defines the name of the header + used to group incoming requests. + type: string + requestHost: + description: RequestHost defines whether to consider the request + Host as the source. + type: boolean + type: object + type: object + ipWhiteList: + description: 'IPWhiteList holds the IP whitelist middleware configuration. + This middleware accepts / refuses requests based on the client IP. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/ipwhitelist/' + properties: + ipStrategy: + description: 'IPStrategy holds the IP strategy configuration used + by Traefik to determine the client IP. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/ipwhitelist/#ipstrategy' + properties: + depth: + description: Depth tells Traefik to use the X-Forwarded-For + header and take the IP located at the depth position (starting + from the right). + type: integer + excludedIPs: + description: ExcludedIPs configures Traefik to scan the X-Forwarded-For + header and select the first IP not in the list. + items: + type: string + type: array + type: object + sourceRange: + description: SourceRange defines the set of allowed IPs (or ranges + of allowed IPs by using CIDR notation). + items: + type: string + type: array + type: object + passTLSClientCert: + description: 'PassTLSClientCert holds the pass TLS client cert middleware + configuration. This middleware adds the selected data from the passed + client TLS certificate to a header. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/passtlsclientcert/' + properties: + info: + description: Info selects the specific client certificate details + you want to add to the X-Forwarded-Tls-Client-Cert-Info header. + properties: + issuer: + description: Issuer defines the client certificate issuer + details to add to the X-Forwarded-Tls-Client-Cert-Info header. + properties: + commonName: + description: CommonName defines whether to add the organizationalUnit + information into the issuer. + type: boolean + country: + description: Country defines whether to add the country + information into the issuer. + type: boolean + domainComponent: + description: DomainComponent defines whether to add the + domainComponent information into the issuer. + type: boolean + locality: + description: Locality defines whether to add the locality + information into the issuer. + type: boolean + organization: + description: Organization defines whether to add the organization + information into the issuer. + type: boolean + province: + description: Province defines whether to add the province + information into the issuer. + type: boolean + serialNumber: + description: SerialNumber defines whether to add the serialNumber + information into the issuer. + type: boolean + type: object + notAfter: + description: NotAfter defines whether to add the Not After + information from the Validity part. + type: boolean + notBefore: + description: NotBefore defines whether to add the Not Before + information from the Validity part. + type: boolean + sans: + description: Sans defines whether to add the Subject Alternative + Name information from the Subject Alternative Name part. + type: boolean + serialNumber: + description: SerialNumber defines whether to add the client + serialNumber information. + type: boolean + subject: + description: Subject defines the client certificate subject + details to add to the X-Forwarded-Tls-Client-Cert-Info header. + properties: + commonName: + description: CommonName defines whether to add the organizationalUnit + information into the subject. + type: boolean + country: + description: Country defines whether to add the country + information into the subject. + type: boolean + domainComponent: + description: DomainComponent defines whether to add the + domainComponent information into the subject. + type: boolean + locality: + description: Locality defines whether to add the locality + information into the subject. + type: boolean + organization: + description: Organization defines whether to add the organization + information into the subject. + type: boolean + organizationalUnit: + description: OrganizationalUnit defines whether to add + the organizationalUnit information into the subject. + type: boolean + province: + description: Province defines whether to add the province + information into the subject. + type: boolean + serialNumber: + description: SerialNumber defines whether to add the serialNumber + information into the subject. + type: boolean + type: object + type: object + pem: + description: PEM sets the X-Forwarded-Tls-Client-Cert header with + the certificate. + type: boolean + type: object + plugin: + additionalProperties: + x-kubernetes-preserve-unknown-fields: true + description: 'Plugin defines the middleware plugin configuration. + More info: https://doc.traefik.io/traefik/plugins/' + type: object + rateLimit: + description: 'RateLimit holds the rate limit configuration. This middleware + ensures that services will receive a fair amount of requests, and + allows one to define what fair is. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/ratelimit/' + properties: + average: + description: Average is the maximum rate, by default in requests/s, + allowed for the given source. It defaults to 0, which means + no rate limiting. The rate is actually defined by dividing Average + by Period. So for a rate below 1req/s, one needs to define a + Period larger than a second. + format: int64 + type: integer + burst: + description: Burst is the maximum number of requests allowed to + arrive in the same arbitrarily small period of time. It defaults + to 1. + format: int64 + type: integer + period: + anyOf: + - type: integer + - type: string + description: 'Period, in combination with Average, defines the + actual maximum rate, such as: r = Average / Period. It defaults + to a second.' + x-kubernetes-int-or-string: true + sourceCriterion: + description: SourceCriterion defines what criterion is used to + group requests as originating from a common source. If several + strategies are defined at the same time, an error will be raised. + If none are set, the default is to use the request's remote + address field (as an ipStrategy). + properties: + ipStrategy: + description: 'IPStrategy holds the IP strategy configuration + used by Traefik to determine the client IP. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/ipwhitelist/#ipstrategy' + properties: + depth: + description: Depth tells Traefik to use the X-Forwarded-For + header and take the IP located at the depth position + (starting from the right). + type: integer + excludedIPs: + description: ExcludedIPs configures Traefik to scan the + X-Forwarded-For header and select the first IP not in + the list. + items: + type: string + type: array + type: object + requestHeaderName: + description: RequestHeaderName defines the name of the header + used to group incoming requests. + type: string + requestHost: + description: RequestHost defines whether to consider the request + Host as the source. + type: boolean + type: object + type: object + redirectRegex: + description: 'RedirectRegex holds the redirect regex middleware configuration. + This middleware redirects a request using regex matching and replacement. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/redirectregex/#regex' + properties: + permanent: + description: Permanent defines whether the redirection is permanent + (301). + type: boolean + regex: + description: Regex defines the regex used to match and capture + elements from the request URL. + type: string + replacement: + description: Replacement defines how to modify the URL to have + the new target URL. + type: string + type: object + redirectScheme: + description: 'RedirectScheme holds the redirect scheme middleware + configuration. This middleware redirects requests from a scheme/port + to another. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/redirectscheme/' + properties: + permanent: + description: Permanent defines whether the redirection is permanent + (301). + type: boolean + port: + description: Port defines the port of the new URL. + type: string + scheme: + description: Scheme defines the scheme of the new URL. + type: string + type: object + replacePath: + description: 'ReplacePath holds the replace path middleware configuration. + This middleware replaces the path of the request URL and store the + original path in an X-Replaced-Path header. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/replacepath/' + properties: + path: + description: Path defines the path to use as replacement in the + request URL. + type: string + type: object + replacePathRegex: + description: 'ReplacePathRegex holds the replace path regex middleware + configuration. This middleware replaces the path of a URL using + regex matching and replacement. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/replacepathregex/' + properties: + regex: + description: Regex defines the regular expression used to match + and capture the path from the request URL. + type: string + replacement: + description: Replacement defines the replacement path format, + which can include captured variables. + type: string + type: object + retry: + description: 'Retry holds the retry middleware configuration. This + middleware reissues requests a given number of times to a backend + server if that server does not reply. As soon as the server answers, + the middleware stops retrying, regardless of the response status. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/retry/' + properties: + attempts: + description: Attempts defines how many times the request should + be retried. + type: integer + initialInterval: + anyOf: + - type: integer + - type: string + description: InitialInterval defines the first wait time in the + exponential backoff series. The maximum interval is calculated + as twice the initialInterval. If unspecified, requests will + be retried immediately. The value of initialInterval should + be provided in seconds or as a valid duration format, see https://pkg.go.dev/time#ParseDuration. + x-kubernetes-int-or-string: true + type: object + stripPrefix: + description: 'StripPrefix holds the strip prefix middleware configuration. + This middleware removes the specified prefixes from the URL path. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/stripprefix/' + properties: + forceSlash: + description: 'ForceSlash ensures that the resulting stripped path + is not the empty string, by replacing it with / when necessary. + Default: true.' + type: boolean + prefixes: + description: Prefixes defines the prefixes to strip from the request + URL. + items: + type: string + type: array + type: object + stripPrefixRegex: + description: 'StripPrefixRegex holds the strip prefix regex middleware + configuration. This middleware removes the matching prefixes from + the URL path. More info: https://doc.traefik.io/traefik/v2.9/middlewares/http/stripprefixregex/' + properties: + regex: + description: Regex defines the regular expression to match the + path prefix from the request URL. + items: + type: string + type: array + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: middlewaretcps.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: MiddlewareTCP + listKind: MiddlewareTCPList + plural: middlewaretcps + singular: middlewaretcp + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'MiddlewareTCP is the CRD implementation of a Traefik TCP middleware. + More info: https://doc.traefik.io/traefik/v2.9/middlewares/overview/' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MiddlewareTCPSpec defines the desired state of a MiddlewareTCP. + properties: + inFlightConn: + description: InFlightConn defines the InFlightConn middleware configuration. + properties: + amount: + description: Amount defines the maximum amount of allowed simultaneous + connections. The middleware closes the connection if there are + already amount connections opened. + format: int64 + type: integer + type: object + ipWhiteList: + description: IPWhiteList defines the IPWhiteList middleware configuration. + properties: + sourceRange: + description: SourceRange defines the allowed IPs (or ranges of + allowed IPs by using CIDR notation). + items: + type: string + type: array + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: serverstransports.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: ServersTransport + listKind: ServersTransportList + plural: serverstransports + singular: serverstransport + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'ServersTransport is the CRD implementation of a ServersTransport. + If no serversTransport is specified, the default@internal will be used. + The default@internal serversTransport is created from the static configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/services/#serverstransport_1' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ServersTransportSpec defines the desired state of a ServersTransport. + properties: + certificatesSecrets: + description: CertificatesSecrets defines a list of secret storing + client certificates for mTLS. + items: + type: string + type: array + disableHTTP2: + description: DisableHTTP2 disables HTTP/2 for connections with backend + servers. + type: boolean + forwardingTimeouts: + description: ForwardingTimeouts defines the timeouts for requests + forwarded to the backend servers. + properties: + dialTimeout: + anyOf: + - type: integer + - type: string + description: DialTimeout is the amount of time to wait until a + connection to a backend server can be established. + x-kubernetes-int-or-string: true + idleConnTimeout: + anyOf: + - type: integer + - type: string + description: IdleConnTimeout is the maximum period for which an + idle HTTP keep-alive connection will remain open before closing + itself. + x-kubernetes-int-or-string: true + pingTimeout: + anyOf: + - type: integer + - type: string + description: PingTimeout is the timeout after which the HTTP/2 + connection will be closed if a response to ping is not received. + x-kubernetes-int-or-string: true + readIdleTimeout: + anyOf: + - type: integer + - type: string + description: ReadIdleTimeout is the timeout after which a health + check using ping frame will be carried out if no frame is received + on the HTTP/2 connection. + x-kubernetes-int-or-string: true + responseHeaderTimeout: + anyOf: + - type: integer + - type: string + description: ResponseHeaderTimeout is the amount of time to wait + for a server's response headers after fully writing the request + (including its body, if any). + x-kubernetes-int-or-string: true + type: object + insecureSkipVerify: + description: InsecureSkipVerify disables SSL certificate verification. + type: boolean + maxIdleConnsPerHost: + description: MaxIdleConnsPerHost controls the maximum idle (keep-alive) + to keep per-host. + type: integer + peerCertURI: + description: PeerCertURI defines the peer cert URI used to match against + SAN URI during the peer certificate verification. + type: string + rootCAsSecrets: + description: RootCAsSecrets defines a list of CA secret used to validate + self-signed certificate. + items: + type: string + type: array + serverName: + description: ServerName defines the server name used to contact the + server. + type: string + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: tlsoptions.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: TLSOption + listKind: TLSOptionList + plural: tlsoptions + singular: tlsoption + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'TLSOption is the CRD implementation of a Traefik TLS Option, + allowing to configure some parameters of the TLS connection. More info: + https://doc.traefik.io/traefik/v2.9/https/tls/#tls-options' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TLSOptionSpec defines the desired state of a TLSOption. + properties: + alpnProtocols: + description: 'ALPNProtocols defines the list of supported application + level protocols for the TLS handshake, in order of preference. More + info: https://doc.traefik.io/traefik/v2.9/https/tls/#alpn-protocols' + items: + type: string + type: array + cipherSuites: + description: 'CipherSuites defines the list of supported cipher suites + for TLS versions up to TLS 1.2. More info: https://doc.traefik.io/traefik/v2.9/https/tls/#cipher-suites' + items: + type: string + type: array + clientAuth: + description: ClientAuth defines the server's policy for TLS Client + Authentication. + properties: + clientAuthType: + description: ClientAuthType defines the client authentication + type to apply. + enum: + - NoClientCert + - RequestClientCert + - RequireAnyClientCert + - VerifyClientCertIfGiven + - RequireAndVerifyClientCert + type: string + secretNames: + description: SecretNames defines the names of the referenced Kubernetes + Secret storing certificate details. + items: + type: string + type: array + type: object + curvePreferences: + description: 'CurvePreferences defines the preferred elliptic curves + in a specific order. More info: https://doc.traefik.io/traefik/v2.9/https/tls/#curve-preferences' + items: + type: string + type: array + maxVersion: + description: 'MaxVersion defines the maximum TLS version that Traefik + will accept. Possible values: VersionTLS10, VersionTLS11, VersionTLS12, + VersionTLS13. Default: None.' + type: string + minVersion: + description: 'MinVersion defines the minimum TLS version that Traefik + will accept. Possible values: VersionTLS10, VersionTLS11, VersionTLS12, + VersionTLS13. Default: VersionTLS10.' + type: string + preferServerCipherSuites: + description: 'PreferServerCipherSuites defines whether the server + chooses a cipher suite among his own instead of among the client''s. + It is enabled automatically when minVersion or maxVersion is set. + Deprecated: https://github.com/golang/go/issues/45430' + type: boolean + sniStrict: + description: SniStrict defines whether Traefik allows connections + from clients connections that do not specify a server_name extension. + type: boolean + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: tlsstores.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: TLSStore + listKind: TLSStoreList + plural: tlsstores + singular: tlsstore + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'TLSStore is the CRD implementation of a Traefik TLS Store. For + the time being, only the TLSStore named default is supported. This means + that you cannot have two stores that are named default in different Kubernetes + namespaces. More info: https://doc.traefik.io/traefik/v2.9/https/tls/#certificates-stores' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TLSStoreSpec defines the desired state of a TLSStore. + properties: + certificates: + description: Certificates is a list of secret names, each secret holding + a key/certificate pair to add to the store. + items: + description: Certificate holds a secret name for the TLSStore resource. + properties: + secretName: + description: SecretName is the name of the referenced Kubernetes + Secret to specify the certificate details. + type: string + required: + - secretName + type: object + type: array + defaultCertificate: + description: DefaultCertificate defines the default certificate configuration. + properties: + secretName: + description: SecretName is the name of the referenced Kubernetes + Secret to specify the certificate details. + type: string + required: + - secretName + type: object + defaultGeneratedCert: + description: DefaultGeneratedCert defines the default generated certificate + configuration. + properties: + domain: + description: Domain is the domain definition for the DefaultCertificate. + properties: + main: + description: Main defines the main domain name. + type: string + sans: + description: SANs defines the subject alternative domain names. + items: + type: string + type: array + type: object + resolver: + description: Resolver is the name of the resolver that will be + used to issue the DefaultCertificate. + type: string + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: traefikservices.traefik.containo.us +spec: + group: traefik.containo.us + names: + kind: TraefikService + listKind: TraefikServiceList + plural: traefikservices + singular: traefikservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: 'TraefikService is the CRD implementation of a Traefik Service. + TraefikService object allows to: - Apply weight to Services on load-balancing + - Mirror traffic on services More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#kind-traefikservice' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TraefikServiceSpec defines the desired state of a TraefikService. + properties: + mirroring: + description: Mirroring defines the Mirroring service configuration. + properties: + kind: + description: Kind defines the kind of the Service. + enum: + - Service + - TraefikService + type: string + maxBodySize: + description: MaxBodySize defines the maximum size allowed for + the body of the request. If the body is larger, the request + is not mirrored. Default value is -1, which means unlimited + size. + format: int64 + type: integer + mirrors: + description: Mirrors defines the list of mirrors where Traefik + will duplicate the traffic. + items: + description: MirrorService holds the mirror configuration. + properties: + kind: + description: Kind defines the kind of the Service. + enum: + - Service + - TraefikService + type: string + name: + description: Name defines the name of the referenced Kubernetes + Service or TraefikService. The differentiation between + the two is specified in the Kind field. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service or TraefikService. + type: string + passHostHeader: + description: PassHostHeader defines whether the client Host + header is forwarded to the upstream Kubernetes Service. + By default, passHostHeader is true. + type: boolean + percent: + description: 'Percent defines the part of the traffic to + mirror. Supported values: 0 to 100.' + type: integer + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. + This can be a reference to a named port. + x-kubernetes-int-or-string: true + responseForwarding: + description: ResponseForwarding defines how Traefik forwards + the response from the upstream Kubernetes Service to the + client. + properties: + flushInterval: + description: 'FlushInterval defines the interval, in + milliseconds, in between flushes to the client while + copying the response body. A negative value means + to flush immediately after each write to the client. + This configuration is ignored when ReverseProxy recognizes + a response as a streaming response; for such responses, + writes are flushed to the client immediately. Default: + 100ms' + type: string + type: object + scheme: + description: Scheme defines the scheme to use for the request + to the upstream Kubernetes Service. It defaults to https + when Kubernetes Service port is 443, http otherwise. + type: string + serversTransport: + description: ServersTransport defines the name of ServersTransport + resource to use. It allows to configure the transport + between Traefik and your servers. Can only be used on + a Kubernetes Service. + type: string + sticky: + description: 'Sticky defines the sticky sessions configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/services/#sticky-sessions' + properties: + cookie: + description: Cookie defines the sticky cookie configuration. + properties: + httpOnly: + description: HTTPOnly defines whether the cookie + can be accessed by client-side APIs, such as JavaScript. + type: boolean + name: + description: Name defines the Cookie name. + type: string + sameSite: + description: 'SameSite defines the same site policy. + More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite' + type: string + secure: + description: Secure defines whether the cookie can + only be transmitted over an encrypted connection + (i.e. HTTPS). + type: boolean + type: object + type: object + strategy: + description: Strategy defines the load balancing strategy + between the servers. RoundRobin is the only supported + value at the moment. + type: string + weight: + description: Weight defines the weight and should only be + specified when Name references a TraefikService object + (and to be precise, one that embeds a Weighted Round Robin). + type: integer + required: + - name + type: object + type: array + name: + description: Name defines the name of the referenced Kubernetes + Service or TraefikService. The differentiation between the two + is specified in the Kind field. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service or TraefikService. + type: string + passHostHeader: + description: PassHostHeader defines whether the client Host header + is forwarded to the upstream Kubernetes Service. By default, + passHostHeader is true. + type: boolean + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. This + can be a reference to a named port. + x-kubernetes-int-or-string: true + responseForwarding: + description: ResponseForwarding defines how Traefik forwards the + response from the upstream Kubernetes Service to the client. + properties: + flushInterval: + description: 'FlushInterval defines the interval, in milliseconds, + in between flushes to the client while copying the response + body. A negative value means to flush immediately after + each write to the client. This configuration is ignored + when ReverseProxy recognizes a response as a streaming response; + for such responses, writes are flushed to the client immediately. + Default: 100ms' + type: string + type: object + scheme: + description: Scheme defines the scheme to use for the request + to the upstream Kubernetes Service. It defaults to https when + Kubernetes Service port is 443, http otherwise. + type: string + serversTransport: + description: ServersTransport defines the name of ServersTransport + resource to use. It allows to configure the transport between + Traefik and your servers. Can only be used on a Kubernetes Service. + type: string + sticky: + description: 'Sticky defines the sticky sessions configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/services/#sticky-sessions' + properties: + cookie: + description: Cookie defines the sticky cookie configuration. + properties: + httpOnly: + description: HTTPOnly defines whether the cookie can be + accessed by client-side APIs, such as JavaScript. + type: boolean + name: + description: Name defines the Cookie name. + type: string + sameSite: + description: 'SameSite defines the same site policy. More + info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite' + type: string + secure: + description: Secure defines whether the cookie can only + be transmitted over an encrypted connection (i.e. HTTPS). + type: boolean + type: object + type: object + strategy: + description: Strategy defines the load balancing strategy between + the servers. RoundRobin is the only supported value at the moment. + type: string + weight: + description: Weight defines the weight and should only be specified + when Name references a TraefikService object (and to be precise, + one that embeds a Weighted Round Robin). + type: integer + required: + - name + type: object + weighted: + description: Weighted defines the Weighted Round Robin configuration. + properties: + services: + description: Services defines the list of Kubernetes Service and/or + TraefikService to load-balance, with weight. + items: + description: Service defines an upstream HTTP service to proxy + traffic to. + properties: + kind: + description: Kind defines the kind of the Service. + enum: + - Service + - TraefikService + type: string + name: + description: Name defines the name of the referenced Kubernetes + Service or TraefikService. The differentiation between + the two is specified in the Kind field. + type: string + namespace: + description: Namespace defines the namespace of the referenced + Kubernetes Service or TraefikService. + type: string + passHostHeader: + description: PassHostHeader defines whether the client Host + header is forwarded to the upstream Kubernetes Service. + By default, passHostHeader is true. + type: boolean + port: + anyOf: + - type: integer + - type: string + description: Port defines the port of a Kubernetes Service. + This can be a reference to a named port. + x-kubernetes-int-or-string: true + responseForwarding: + description: ResponseForwarding defines how Traefik forwards + the response from the upstream Kubernetes Service to the + client. + properties: + flushInterval: + description: 'FlushInterval defines the interval, in + milliseconds, in between flushes to the client while + copying the response body. A negative value means + to flush immediately after each write to the client. + This configuration is ignored when ReverseProxy recognizes + a response as a streaming response; for such responses, + writes are flushed to the client immediately. Default: + 100ms' + type: string + type: object + scheme: + description: Scheme defines the scheme to use for the request + to the upstream Kubernetes Service. It defaults to https + when Kubernetes Service port is 443, http otherwise. + type: string + serversTransport: + description: ServersTransport defines the name of ServersTransport + resource to use. It allows to configure the transport + between Traefik and your servers. Can only be used on + a Kubernetes Service. + type: string + sticky: + description: 'Sticky defines the sticky sessions configuration. + More info: https://doc.traefik.io/traefik/v2.9/routing/services/#sticky-sessions' + properties: + cookie: + description: Cookie defines the sticky cookie configuration. + properties: + httpOnly: + description: HTTPOnly defines whether the cookie + can be accessed by client-side APIs, such as JavaScript. + type: boolean + name: + description: Name defines the Cookie name. + type: string + sameSite: + description: 'SameSite defines the same site policy. + More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite' + type: string + secure: + description: Secure defines whether the cookie can + only be transmitted over an encrypted connection + (i.e. HTTPS). + type: boolean + type: object + type: object + strategy: + description: Strategy defines the load balancing strategy + between the servers. RoundRobin is the only supported + value at the moment. + type: string + weight: + description: Weight defines the weight and should only be + specified when Name references a TraefikService object + (and to be precise, one that embeds a Weighted Round Robin). + type: integer + required: + - name + type: object + type: array + sticky: + description: 'Sticky defines whether sticky sessions are enabled. + More info: https://doc.traefik.io/traefik/v2.9/routing/providers/kubernetes-crd/#stickiness-and-load-balancing' + properties: + cookie: + description: Cookie defines the sticky cookie configuration. + properties: + httpOnly: + description: HTTPOnly defines whether the cookie can be + accessed by client-side APIs, such as JavaScript. + type: boolean + name: + description: Name defines the Cookie name. + type: string + sameSite: + description: 'SameSite defines the same site policy. More + info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite' + type: string + secure: + description: Secure defines whether the cookie can only + be transmitted over an encrypted connection (i.e. HTTPS). + type: boolean + type: object + type: object + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/k8s/traefik-rbac.yaml b/k8s/traefik-rbac.yaml new file mode 100644 index 0000000..180f6ba --- /dev/null +++ b/k8s/traefik-rbac.yaml @@ -0,0 +1,72 @@ +# Based on traefik/traefik helm chart 20.8.0 for traefik v2.9.6 + +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + namespace: traefik + name: traefik + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: traefik +rules: + - apiGroups: + - extensions + - networking.k8s.io + resources: + - ingressclasses + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - traefik.containo.us + resources: + - ingressroutes + - ingressroutetcps + - ingressrouteudps + - middlewares + - middlewaretcps + - tlsoptions + - tlsstores + - traefikservices + - serverstransports + verbs: + - get + - list + - watch + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: traefik +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: traefik +subjects: + - namespace: traefik + kind: ServiceAccount + name: traefik diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml new file mode 100644 index 0000000..d7bee5d --- /dev/null +++ b/k8s/traefik.yaml @@ -0,0 +1,145 @@ +# Based on traefik/traefik helm chart 20.8.0 for traefik v2.9.6 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: traefik + name: traefik +spec: + replicas: 1 + selector: + matchLabels: + app: traefik + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + minReadySeconds: 0 + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "9100" + labels: + app: traefik + spec: + serviceAccountName: traefik + terminationGracePeriodSeconds: 60 + hostNetwork: false + containers: + - image: traefik:v2.9.6 + imagePullPolicy: IfNotPresent + name: traefik + resources: {} + readinessProbe: + httpGet: + path: /ping + port: 9000 + scheme: HTTP + failureThreshold: 1 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /ping + port: 9000 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + ports: + - name: traefik + containerPort: 9000 + - name: web + containerPort: 8000 + - name: websecure + containerPort: 8443 + - name: metrics + containerPort: 9100 + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - name: data + mountPath: /data + - name: tmp + mountPath: /tmp + args: + - "--global.checknewversion" + - "--global.sendanonymoususage" + - "--entrypoints.metrics.address=:9100/tcp" + - "--entrypoints.traefik.address=:9000/tcp" + - "--entrypoints.web.address=:8000/tcp" + - "--entrypoints.websecure.address=:8443/tcp" + - "--api.dashboard=true" + - "--ping=true" + - "--metrics.prometheus=true" + - "--metrics.prometheus.entrypoint=metrics" + - "--providers.kubernetescrd" + - "--providers.kubernetesingress" + - "--entrypoints.websecure.http.tls=true" + volumes: + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} + securityContext: + fsGroup: 65532 + +--- +kind: Service +apiVersion: v1 +metadata: + namespace: traefik + name: traefik +spec: + type: LoadBalancer + selector: + app: traefik + ports: + - port: 80 + name: "web" + targetPort: web + protocol: TCP + - port: 443 + name: "websecure" + targetPort: websecure + protocol: TCP + +--- +kind: IngressClass +apiVersion: networking.k8s.io/v1 +metadata: + name: traefik + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: traefik.io/ingress-controller +--- + +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 +metadata: + namespace: traefik + name: traefik-dashboard +spec: + entryPoints: + - traefik + routes: + - match: PathPrefix(`/dashboard`) || PathPrefix(`/api`) + kind: Rule + services: + - name: api@internal + kind: TraefikService diff --git a/tools/template/Makefile b/tools/template/Makefile new file mode 100644 index 0000000..6b3dfec --- /dev/null +++ b/tools/template/Makefile @@ -0,0 +1,5 @@ +bin: + mkdir bin + +bin/template: bin go.mod go.sum template.go + go build -o bin/template template.go diff --git a/tools/template/go.mod b/tools/template/go.mod new file mode 100644 index 0000000..bf84ca2 --- /dev/null +++ b/tools/template/go.mod @@ -0,0 +1,8 @@ +module github.com/raxod502/riju/tools/go + +go 1.16 + +require ( + github.com/Masterminds/sprig/v3 v3.2.2 // indirect + gopkg.in/yaml.v2 v2.4.0 +) diff --git a/tools/template/go.sum b/tools/template/go.sum new file mode 100644 index 0000000..5ad2ee8 --- /dev/null +++ b/tools/template/go.sum @@ -0,0 +1,39 @@ +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= +github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= +github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8= +golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/template/template b/tools/template/template new file mode 100755 index 0000000..d52b63d --- /dev/null +++ b/tools/template/template @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "$(dirname "$0")" +make -s bin/template +exec bin/template diff --git a/tools/template/template.go b/tools/template/template.go new file mode 100644 index 0000000..b863108 --- /dev/null +++ b/tools/template/template.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/Masterminds/sprig/v3" + "gopkg.in/yaml.v2" +) + +func findProjectRoot() (string, error) { + curDir, err := os.Getwd() + if err != nil { + return "", err + } + prevDir := "" + for curDir != prevDir { + if _, err := os.Stat(filepath.Join(curDir, ".git")); err == nil { + return curDir, nil + } else if !os.IsNotExist(err) { + return "", err + } + prevDir = curDir + curDir = filepath.Dir(curDir) + } + return "", fmt.Errorf("not a git repository (or any of the parent directories): .git") +} + +func mainInternal() error { + projectRoot, err := findProjectRoot() + if err != nil { + return err + } + envText, err := os.ReadFile(filepath.Join(projectRoot, "env.yaml")) + if err != nil { + return err + } + var env interface{} + if err := yaml.Unmarshal(envText, &env); err != nil { + return err + } + if err := filepath.WalkDir(projectRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.Name() == ".git" { + return fs.SkipDir + } + dir := filepath.Dir(path) + if strings.Contains(d.Name(), ".in.") { + fmt.Fprintf(os.Stderr, "template_secrets.go: processing %s\n", path) + input, err := os.ReadFile(path) + if err != nil { + return err + } + tmpl, err := template.New("").Funcs(sprig.TxtFuncMap()).Option("missingkey=error").Parse(string(input)) + if err != nil { + return err + } + var output bytes.Buffer + if err := tmpl.Execute(&output, env); err != nil { + return err + } + if err := os.WriteFile( + filepath.Join(dir, strings.ReplaceAll(d.Name(), ".in.", ".out.")), + output.Bytes(), 0644, + ); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + return nil +} + +func main() { + if err := mainInternal(); err != nil { + fmt.Fprintf(os.Stderr, "template.go: %s\n", err.Error()) + os.Exit(1) + } +} From 7675d36df165e09ae739aed788f80818fe51500d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 19 Dec 2022 15:24:57 -0700 Subject: [PATCH 095/123] Move traefik config to configmap --- k8s/traefik-config.yaml | 28 ++++++++++++++++++++++++++++ k8s/traefik.yaml | 19 +++++-------------- 2 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 k8s/traefik-config.yaml diff --git a/k8s/traefik-config.yaml b/k8s/traefik-config.yaml new file mode 100644 index 0000000..8e5efc2 --- /dev/null +++ b/k8s/traefik-config.yaml @@ -0,0 +1,28 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + namespace: traefik + name: traefik-config +data: + traefik.yaml: | + entrypoints: + traefik: + address: ":9000/tcp" + metrics: + address: ":9100/tcp" + web: + address: ":8000/tcp" + websecure: + address: ":8443/tcp" + http: + tls: true + api: + dashboard: true + ping: true + metrics: + prometheus: + entrypoint: metrics + providers: + kubernetescrd: true + kubernetesingress: true diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index d7bee5d..5c654a8 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -72,25 +72,16 @@ spec: runAsNonRoot: true runAsUser: 65532 volumeMounts: + - name: config + mountPath: /etc/traefik - name: data mountPath: /data - name: tmp mountPath: /tmp - args: - - "--global.checknewversion" - - "--global.sendanonymoususage" - - "--entrypoints.metrics.address=:9100/tcp" - - "--entrypoints.traefik.address=:9000/tcp" - - "--entrypoints.web.address=:8000/tcp" - - "--entrypoints.websecure.address=:8443/tcp" - - "--api.dashboard=true" - - "--ping=true" - - "--metrics.prometheus=true" - - "--metrics.prometheus.entrypoint=metrics" - - "--providers.kubernetescrd" - - "--providers.kubernetesingress" - - "--entrypoints.websecure.http.tls=true" volumes: + - name: config + configMap: + name: traefik-config - name: data emptyDir: {} - name: tmp From 8dd4b72fc7be18445fe2722b65aa67a3aeb3e3b5 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 19 Dec 2022 15:48:50 -0700 Subject: [PATCH 096/123] Fix some thing --- env.yaml.bash | 3 +++ k8s/traefik-config.in.yaml | 33 +++++++++++++++++++++++++++++++ k8s/traefik-config.yaml | 28 -------------------------- k8s/traefik.yaml | 40 ++++++++++++-------------------------- 4 files changed, 48 insertions(+), 56 deletions(-) create mode 100644 k8s/traefik-config.in.yaml delete mode 100644 k8s/traefik-config.yaml diff --git a/env.yaml.bash b/env.yaml.bash index 4720096..e22cdb6 100755 --- a/env.yaml.bash +++ b/env.yaml.bash @@ -11,6 +11,9 @@ networking: domain: riju.example.com # FIXME ip: x.y.z.w # FIXME +contact: + letsEncryptEmail: ops@example.com # FIXME + metallb: secretkey: "$(pwgen -s 256 1)" diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml new file mode 100644 index 0000000..ec668cd --- /dev/null +++ b/k8s/traefik-config.in.yaml @@ -0,0 +1,33 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + namespace: traefik + name: traefik-config +data: + traefik.yaml: | + entrypoints: + http: + address: ":8000/tcp" + https: + address: ":8443/tcp" + http: + tls: true + healthcheck: + address: ":9000/tcp" + docker: + address: ":31000/tcp" + http: + tls: true + ping: + entryPoint: "healthcheck" + providers: + kubernetescrd: true + kubernetesingress: true + certificatesResolvers: + riju: + acme: + email: "{{ .contact.letsEncryptEmail }}" + storage: acme.json + httpChallenge: + entryPoint: http diff --git a/k8s/traefik-config.yaml b/k8s/traefik-config.yaml deleted file mode 100644 index 8e5efc2..0000000 --- a/k8s/traefik-config.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -kind: ConfigMap -apiVersion: v1 -metadata: - namespace: traefik - name: traefik-config -data: - traefik.yaml: | - entrypoints: - traefik: - address: ":9000/tcp" - metrics: - address: ":9100/tcp" - web: - address: ":8000/tcp" - websecure: - address: ":8443/tcp" - http: - tls: true - api: - dashboard: true - ping: true - metrics: - prometheus: - entrypoint: metrics - providers: - kubernetescrd: true - kubernetesingress: true diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index 5c654a8..4a9f4df 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -55,14 +55,14 @@ spec: successThreshold: 1 timeoutSeconds: 2 ports: - - name: traefik - containerPort: 9000 - - name: web + - name: http containerPort: 8000 - - name: websecure + - name: https containerPort: 8443 - - name: metrics - containerPort: 9100 + - name: ping + containerPort: 9000 + - name: docker + containerPort: 31000 securityContext: capabilities: drop: @@ -101,13 +101,13 @@ spec: app: traefik ports: - port: 80 - name: "web" - targetPort: web - protocol: TCP + name: http + targetPort: 8000 - port: 443 - name: "websecure" - targetPort: websecure - protocol: TCP + name: https + targetPort: 8443 + - port: 31000 + name: docker --- kind: IngressClass @@ -118,19 +118,3 @@ metadata: ingressclass.kubernetes.io/is-default-class: "true" spec: controller: traefik.io/ingress-controller ---- - -kind: IngressRoute -apiVersion: traefik.containo.us/v1alpha1 -metadata: - namespace: traefik - name: traefik-dashboard -spec: - entryPoints: - - traefik - routes: - - match: PathPrefix(`/dashboard`) || PathPrefix(`/api`) - kind: Rule - services: - - name: api@internal - kind: TraefikService From 5448cfc93b0ed3475c66ba85d666b899acde458e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Mon, 19 Dec 2022 21:54:40 -0700 Subject: [PATCH 097/123] Use PVC for Traefik ACME data --- env.yaml.bash | 1 + k8s/riju.yaml | 3 --- k8s/traefik-config.in.yaml | 10 +++++++++- k8s/traefik.yaml | 18 ++++++++++++++++-- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/env.yaml.bash b/env.yaml.bash index e22cdb6..9bdd552 100755 --- a/env.yaml.bash +++ b/env.yaml.bash @@ -13,6 +13,7 @@ networking: contact: letsEncryptEmail: ops@example.com # FIXME + letsEncryptProductionEnabled: false metallb: secretkey: "$(pwgen -s 256 1)" diff --git a/k8s/riju.yaml b/k8s/riju.yaml index dc25937..66d97a4 100644 --- a/k8s/riju.yaml +++ b/k8s/riju.yaml @@ -119,6 +119,3 @@ spec: - name: http port: 80 targetPort: 6119 - ---- -kind: Ingress diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml index ec668cd..de709d8 100644 --- a/k8s/traefik-config.in.yaml +++ b/k8s/traefik-config.in.yaml @@ -15,19 +15,27 @@ data: tls: true healthcheck: address: ":9000/tcp" + metrics: + address: ":9100/tcp" docker: address: ":31000/tcp" http: tls: true ping: entryPoint: "healthcheck" + metrics: + prometheus: + entryPoint: "metrics" providers: kubernetescrd: true kubernetesingress: true certificatesResolvers: riju: acme: + {{- if not .contact.letsEncryptProductionEnabled }} + caServer: https://acme-staging-v02.api.letsencrypt.org/directory + {{- end }} email: "{{ .contact.letsEncryptEmail }}" - storage: acme.json + storage: /data/acme.json httpChallenge: entryPoint: http diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index 4a9f4df..840a82f 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -1,8 +1,21 @@ # Based on traefik/traefik helm chart 20.8.0 for traefik v2.9.6 --- -apiVersion: apps/v1 +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + namespace: traefik + name: traefik-data +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 128Mi + storageClassName: openebs-hostpath + +--- kind: Deployment +apiVersion: apps/v1 metadata: namespace: traefik name: traefik @@ -83,7 +96,8 @@ spec: configMap: name: traefik-config - name: data - emptyDir: {} + persistentVolumeClaim: + claimName: traefik-data - name: tmp emptyDir: {} securityContext: From 79c72aaac14f2ef6f1ad745c6667d0b325fbc6d3 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 20 Dec 2022 21:35:28 -0700 Subject: [PATCH 098/123] Change more things --- k8s/ingresses.in.yaml | 16 ++++++++++++++++ k8s/traefik-config.in.yaml | 25 +++++++++++++++---------- k8s/traefik.yaml | 10 ++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 k8s/ingresses.in.yaml diff --git a/k8s/ingresses.in.yaml b/k8s/ingresses.in.yaml new file mode 100644 index 0000000..ee2bc74 --- /dev/null +++ b/k8s/ingresses.in.yaml @@ -0,0 +1,16 @@ +--- +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 +metadata: + namespace: riju + name: docker-registry +spec: + entryPoints: + - docker + routes: + - kind: Rule + match: "Host(`*`)" + services: + - namespace: riju + name: docker-registry + port: 80 diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml index de709d8..540346f 100644 --- a/k8s/traefik-config.in.yaml +++ b/k8s/traefik-config.in.yaml @@ -6,29 +6,34 @@ metadata: name: traefik-config data: traefik.yaml: | - entrypoints: + entryPoints: http: - address: ":8000/tcp" + address: ":8000" https: - address: ":8443/tcp" + address: ":8443" http: - tls: true + tls: + certResolver: riju + domains: + - main: k8s.riju.codes healthcheck: - address: ":9000/tcp" + address: ":9000" metrics: - address: ":9100/tcp" + address: ":9100" docker: - address: ":31000/tcp" + address: ":31000" http: - tls: true + tls: + certResolver: riju + domains: + - main: k8s.riju.codes ping: entryPoint: "healthcheck" metrics: prometheus: entryPoint: "metrics" providers: - kubernetescrd: true - kubernetesingress: true + kubernetesCRD: {} certificatesResolvers: riju: acme: diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index 840a82f..d47954c 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -42,6 +42,16 @@ spec: serviceAccountName: traefik terminationGracePeriodSeconds: 60 hostNetwork: false + initContainers: + - name: volume-permissions + image: busybox:1.35 + command: + - "sh" + - "-c" + - "touch /data/acme.json && chmod -Rv 600 /data/* && chown 65532:65532 /data/acme.json" + volumeMounts: + - name: data + mountPath: /data containers: - image: traefik:v2.9.6 imagePullPolicy: IfNotPresent From 7e736b941867cbde806bb029d6852b7c522d47af Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 20 Dec 2022 22:25:57 -0700 Subject: [PATCH 099/123] Holy shit this took so many hours to figure out If you use Host (or HostHeader) it just doesn't work. Lol. Not documented anywhere as far as I can find. --- k8s/ingresses.in.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/ingresses.in.yaml b/k8s/ingresses.in.yaml index ee2bc74..aecc7ea 100644 --- a/k8s/ingresses.in.yaml +++ b/k8s/ingresses.in.yaml @@ -9,7 +9,7 @@ spec: - docker routes: - kind: Rule - match: "Host(`*`)" + match: "PathPrefix(`/`)" services: - namespace: riju name: docker-registry From e0f7f9b0cec9459b076de2f6eb6a53dbcf758831 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 20 Dec 2022 22:29:58 -0700 Subject: [PATCH 100/123] Move things around --- k8s/ingresses.in.yaml | 16 ----- k8s/{riju.yaml => riju-docker-registry.yaml} | 66 ++++---------------- k8s/riju-server.yaml | 56 +++++++++++++++++ 3 files changed, 68 insertions(+), 70 deletions(-) delete mode 100644 k8s/ingresses.in.yaml rename k8s/{riju.yaml => riju-docker-registry.yaml} (55%) create mode 100644 k8s/riju-server.yaml diff --git a/k8s/ingresses.in.yaml b/k8s/ingresses.in.yaml deleted file mode 100644 index aecc7ea..0000000 --- a/k8s/ingresses.in.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -kind: IngressRoute -apiVersion: traefik.containo.us/v1alpha1 -metadata: - namespace: riju - name: docker-registry -spec: - entryPoints: - - docker - routes: - - kind: Rule - match: "PathPrefix(`/`)" - services: - - namespace: riju - name: docker-registry - port: 80 diff --git a/k8s/riju.yaml b/k8s/riju-docker-registry.yaml similarity index 55% rename from k8s/riju.yaml rename to k8s/riju-docker-registry.yaml index 66d97a4..e99fd08 100644 --- a/k8s/riju.yaml +++ b/k8s/riju-docker-registry.yaml @@ -54,68 +54,26 @@ metadata: namespace: riju name: docker-registry spec: - type: NodePort selector: app: docker-registry ports: - name: api port: 80 - nodePort: 30999 targetPort: 5000 --- -kind: Deployment -apiVersion: apps/v1 +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 metadata: namespace: riju - name: riju-server + name: docker-registry spec: - replicas: 1 - selector: - matchLabels: - app: riju-server - template: - metadata: - labels: - app: riju-server - spec: - volumes: - - name: cache - hostPath: - path: /var/cache/riju - - name: docker - hostPath: - path: /var/run/docker.sock - containers: - - name: server - image: "localhost:30999/riju:app" - resources: - requests: - cpu: "1000m" - memory: "6Gi" - limits: - cpu: "1000m" - memory: "6Gi" - ports: - - name: http - containerPort: 6119 - volumeMounts: - - name: cache - mountPath: /var/cache/riju - - name: docker - mountPath: /var/run/docker.sock - readOnly: true - ---- -kind: Service -apiVersion: v1 -metadata: - namespace: riju - name: riju-server -spec: - selector: - app: riju-server - ports: - - name: http - port: 80 - targetPort: 6119 + entryPoints: + - docker + routes: + - kind: Rule + match: "PathPrefix(`/`)" + services: + - namespace: riju + name: docker-registry + port: 80 diff --git a/k8s/riju-server.yaml b/k8s/riju-server.yaml new file mode 100644 index 0000000..0fa545b --- /dev/null +++ b/k8s/riju-server.yaml @@ -0,0 +1,56 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: riju + name: riju-server +spec: + replicas: 1 + selector: + matchLabels: + app: riju-server + template: + metadata: + labels: + app: riju-server + spec: + volumes: + - name: cache + hostPath: + path: /var/cache/riju + - name: docker + hostPath: + path: /var/run/docker.sock + containers: + - name: server + image: "localhost:30999/riju:app" + resources: + requests: + cpu: "1000m" + memory: "6Gi" + limits: + cpu: "1000m" + memory: "6Gi" + ports: + - name: http + containerPort: 6119 + volumeMounts: + - name: cache + mountPath: /var/cache/riju + - name: docker + mountPath: /var/run/docker.sock + readOnly: true + +--- +kind: Service +apiVersion: v1 +metadata: + namespace: riju + name: riju-server +spec: + selector: + app: riju-server + ports: + - name: http + port: 80 + targetPort: 6119 From 48d674a45c08d9f2749c9a1cee698b01ed582301 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 20 Dec 2022 22:48:03 -0700 Subject: [PATCH 101/123] Spin up Docker registry properly, I think --- k8s/riju-docker-registry.yaml | 2 ++ k8s/riju-server.yaml | 10 +++------- k8s/secrets.in.yaml | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/k8s/riju-docker-registry.yaml b/k8s/riju-docker-registry.yaml index e99fd08..4e3df08 100644 --- a/k8s/riju-docker-registry.yaml +++ b/k8s/riju-docker-registry.yaml @@ -56,9 +56,11 @@ metadata: spec: selector: app: docker-registry + type: NodePort ports: - name: api port: 80 + nodePort: 30999 targetPort: 5000 --- diff --git a/k8s/riju-server.yaml b/k8s/riju-server.yaml index 0fa545b..150fa83 100644 --- a/k8s/riju-server.yaml +++ b/k8s/riju-server.yaml @@ -21,16 +21,12 @@ spec: - name: docker hostPath: path: /var/run/docker.sock + imagePullSecrets: + - name: registry-login containers: - name: server image: "localhost:30999/riju:app" - resources: - requests: - cpu: "1000m" - memory: "6Gi" - limits: - cpu: "1000m" - memory: "6Gi" + resources: {} ports: - name: http containerPort: 6119 diff --git a/k8s/secrets.in.yaml b/k8s/secrets.in.yaml index 9ca53a3..65118e8 100644 --- a/k8s/secrets.in.yaml +++ b/k8s/secrets.in.yaml @@ -15,3 +15,22 @@ metadata: name: registry-auth data: htpasswd: "{{ .registry.htpasswd | println | b64enc }}" + +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: riju + name: registry-login +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + { + "auths": { + "localhost:30999": { + "username": "admin", + "password": "{{ .registry.password }}", + "auth": "{{ .registry.password | printf "admin:%s" | b64enc }}" + } + } + } From c7e6a33aeff611e4b08fe54e6e37addb3e015af4 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 20 Dec 2022 22:54:10 -0700 Subject: [PATCH 102/123] Get app server running, technically --- k8s/riju-server.yaml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/k8s/riju-server.yaml b/k8s/riju-server.yaml index 150fa83..34c391d 100644 --- a/k8s/riju-server.yaml +++ b/k8s/riju-server.yaml @@ -27,6 +27,26 @@ spec: - name: server image: "localhost:30999/riju:app" resources: {} + readinessProbe: + httpGet: + path: / + port: 6119 + scheme: HTTP + failureThreshold: 1 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: / + port: 6119 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 ports: - name: http containerPort: 6119 @@ -50,3 +70,20 @@ spec: - name: http port: 80 targetPort: 6119 + +--- +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 +metadata: + namespace: riju + name: riju-server +spec: + entryPoints: + - https + routes: + - kind: Rule + match: "PathPrefix(`/`)" + services: + - namespace: riju + name: riju-server + port: 80 From 331f1c08a1cc0aa816752e87e6d0d72fcaa3dd70 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 21 Dec 2022 12:59:42 -0700 Subject: [PATCH 103/123] Add k8s sdk --- package.json | 1 + yarn.lock | 578 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 574 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index f6163ba..3f7841a 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@babel/parser": "^7.13.11", "@babel/preset-env": "^7.12.11", "@balena/dockerignore": "^1.0.2", + "@kubernetes/client-node": "^0.18.0", "@sentry/node": "^6.11.0", "async-lock": "^1.2.6", "babel-loader": "^8.2.2", diff --git a/yarn.lock b/yarn.lock index 212fcd0..f07fd44 100644 --- a/yarn.lock +++ b/yarn.lock @@ -878,6 +878,32 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.3.tgz#90420f9f9c6d3987f176a19a7d8e764271a2f55d" integrity sha512-Fxt+AfXgjMoin2maPIYzFZnQjAXjAL0PHscM5pRTtatFqB+vZxAM9tLp2Optnuw3QOQC40jTNeGYFOMvyf7v9g== +"@kubernetes/client-node@^0.18.0": + version "0.18.0" + resolved "https://registry.yarnpkg.com/@kubernetes/client-node/-/client-node-0.18.0.tgz#b1002f3c19fb7509ce521ea78f500589f8e3e047" + integrity sha512-Mp6q0OkZQBp+HslIgvHYpsPJk8z6mch231QWtIZQHvs+PaTE6mkUfusYE8fNw3jMjru5mVO/JDz6PTjB9YT2rQ== + dependencies: + "@types/js-yaml" "^4.0.1" + "@types/node" "^10.12.0" + "@types/request" "^2.47.1" + "@types/ws" "^6.0.1" + byline "^5.0.0" + execa "5.0.0" + isomorphic-ws "^4.0.1" + js-yaml "^4.1.0" + jsonpath-plus "^0.19.0" + request "^2.88.0" + rfc4648 "^1.3.0" + shelljs "^0.8.5" + stream-buffers "^3.0.2" + tar "^6.1.11" + tmp-promise "^3.0.2" + tslib "^2.4.1" + underscore "^1.13.6" + ws "^7.3.1" + optionalDependencies: + openid-client "^5.3.0" + "@sentry/core@6.11.0": version "6.11.0" resolved "https://registry.yarnpkg.com/@sentry/core/-/core-6.11.0.tgz#40e94043afcf6407a109be26655c77832c64e740" @@ -946,11 +972,53 @@ "@sentry/types" "6.11.0" tslib "^1.9.3" +"@types/caseless@*": + version "0.12.2" + resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8" + integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w== + +"@types/js-yaml@^4.0.1": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.5.tgz#738dd390a6ecc5442f35e7f03fa1431353f7e138" + integrity sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA== + "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6": version "7.0.7" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== +"@types/node@*": + version "18.11.17" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.17.tgz#5c009e1d9c38f4a2a9d45c0b0c493fe6cdb4bcb5" + integrity sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng== + +"@types/node@^10.12.0": + version "10.17.60" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b" + integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw== + +"@types/request@^2.47.1": + version "2.48.8" + resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.8.tgz#0b90fde3b655ab50976cb8c5ac00faca22f5a82c" + integrity sha512-whjk1EDJPcAR2kYHRbFl/lKeeKYTi05A15K9bnLInCVroNDCtXce57xKdI0/rQaA3K+6q0eFyUBPmqfSndUZdQ== + dependencies: + "@types/caseless" "*" + "@types/node" "*" + "@types/tough-cookie" "*" + form-data "^2.5.0" + +"@types/tough-cookie@*": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" + integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== + +"@types/ws@^6.0.1": + version "6.0.4" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-6.0.4.tgz#7797707c8acce8f76d8c34b370d4645b70421ff1" + integrity sha512-PpPrX7SZW9re6+Ha8ojZG4Se8AZXgf0GK6zmfqEuCsY49LFDNXO3SByp44X3dFEqtB73lkCDAdUazhAjVPiNwg== + dependencies: + "@types/node" "*" + "@webassemblyjs/ast@1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.9.0.tgz#bd850604b4042459a5a41cd7d338cbed695ed964" @@ -1153,7 +1221,7 @@ ajv-keywords@^3.1.0, ajv-keywords@^3.4.1, ajv-keywords@^3.5.2: resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== -ajv@^6.1.0, ajv@^6.10.2, ajv@^6.12.4, ajv@^6.12.5: +ajv@^6.1.0, ajv@^6.10.2, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -1203,6 +1271,11 @@ aproba@^1.1.1: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + arr-diff@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" @@ -1238,6 +1311,18 @@ asn1.js@^5.2.0: minimalistic-assert "^1.0.0" safer-buffer "^2.1.0" +asn1@~0.2.3: + version "0.2.6" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" + integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== + assert@^1.1.1: version "1.5.0" resolved "https://registry.yarnpkg.com/assert/-/assert-1.5.0.tgz#55c109aaf6e0aefdb3dc4b71240c70bf574b18eb" @@ -1271,11 +1356,26 @@ async@^3.2.3: resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g== +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + atob@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA== + +aws4@^1.8.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + babel-loader@^8.2.2: version "8.2.2" resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.2.2.tgz#9363ce84c10c9a40e6c753748e1441b60c8a0b81" @@ -1347,6 +1447,13 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== + dependencies: + tweetnacl "^0.14.3" + big.js@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" @@ -1545,6 +1652,11 @@ builtin-status-codes@^3.0.0: resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8" integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug= +byline@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/byline/-/byline-5.0.0.tgz#741c5216468eadc457b03410118ad77de8c1ddb1" + integrity sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q== + bytes@3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" @@ -1599,6 +1711,11 @@ caniuse-lite@^1.0.30001219: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz#4b7783661515b8e7151fc6376cfd97f0e427b9e5" integrity sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw== +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== + chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1655,6 +1772,11 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + chrome-trace-event@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" @@ -1724,6 +1846,13 @@ colorette@^1.2.1, colorette@^1.2.2: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== +combined-stream@^1.0.6, combined-stream@~1.0.6: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -1828,7 +1957,7 @@ core-js-compat@^3.14.0, core-js-compat@^3.9.1: browserslist "^4.16.6" semver "7.0.0" -core-util-is@~1.0.0: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= @@ -1916,6 +2045,13 @@ cyclist@^1.0.1: resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g== + dependencies: + assert-plus "^1.0.0" + debounce@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.1.tgz#38881d8f4166a5c5848020c11827b834bcb3e0a5" @@ -1976,6 +2112,11 @@ define-property@^2.0.2: is-descriptor "^1.0.2" isobject "^3.0.1" +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -2023,6 +2164,14 @@ duplexify@^3.4.2, duplexify@^3.6.0: readable-stream "^2.0.0" stream-shift "^1.0.0" +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw== + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" @@ -2159,6 +2308,21 @@ evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: md5.js "^1.3.4" safe-buffer "^5.1.1" +execa@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-5.0.0.tgz#4029b0007998a841fbd1032e5f4de86a3c1e3376" + integrity sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ== + dependencies: + cross-spawn "^7.0.3" + get-stream "^6.0.0" + human-signals "^2.1.0" + is-stream "^2.0.0" + merge-stream "^2.0.0" + npm-run-path "^4.0.1" + onetime "^5.1.2" + signal-exit "^3.0.3" + strip-final-newline "^2.0.0" + execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -2245,6 +2409,11 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" +extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + extglob@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" @@ -2259,6 +2428,16 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== + +extsprintf@^1.2.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" + integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== + fast-deep-equal@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -2375,6 +2554,29 @@ for-in@^1.0.2: resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== + +form-data@^2.5.0: + version "2.5.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" + integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" @@ -2410,6 +2612,13 @@ fs-extra@^0.24.0: path-is-absolute "^1.0.0" rimraf "^2.2.8" +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + fs-write-stream-atomic@^1.0.8: version "1.0.10" resolved "https://registry.yarnpkg.com/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz#b47df53493ef911df75731e70a9ded0189db40c9" @@ -2467,6 +2676,13 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng== + dependencies: + assert-plus "^1.0.0" + glob-parent@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae" @@ -2487,6 +2703,18 @@ glob-to-regexp@^0.3.0: resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= +glob@^7.0.0: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + glob@^7.1.3, glob@^7.1.4: version "7.1.7" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" @@ -2509,6 +2737,19 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6 resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q== + +har-validator@~5.1.3: + version "5.1.5" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" + integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== + dependencies: + ajv "^6.12.3" + har-schema "^2.0.0" + has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -2617,6 +2858,15 @@ http-errors@~1.7.2: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ== + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + https-browserify@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" @@ -2698,6 +2948,11 @@ inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +interpret@^1.0.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" + integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== + interpret@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9" @@ -2748,6 +3003,13 @@ is-core-module@^2.2.0: dependencies: has "^1.0.3" +is-core-module@^2.9.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.11.0.tgz#ad4cb3e3863e814523c96f3f58d26cc570ff0144" + integrity sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw== + dependencies: + has "^1.0.3" + is-data-descriptor@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" @@ -2835,6 +3097,11 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== + is-windows@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" @@ -2867,6 +3134,16 @@ isobject@^3.0.0, isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= +isomorphic-ws@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz#55fd4cd6c5e6491e76dc125938dd863f5cd4f2dc" + integrity sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w== + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g== + jake@^10.8.5: version "10.8.5" resolved "https://registry.yarnpkg.com/jake/-/jake-10.8.5.tgz#f2183d2c59382cb274226034543b9c03b8164c46" @@ -2877,11 +3154,28 @@ jake@^10.8.5: filelist "^1.0.1" minimatch "^3.0.4" +jose@^4.10.0: + version "4.11.1" + resolved "https://registry.yarnpkg.com/jose/-/jose-4.11.1.tgz#8f7443549befe5bddcf4bae664a9cbc1a62da4fa" + integrity sha512-YRv4Tk/Wlug8qicwqFNFVEZSdbROCHRAC6qu/i0dyNKr5JQdoa2pIGoS04lLO/jXQX7Z9omoNewYIVIxqZBd9Q== + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg== + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -2902,6 +3196,16 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== +json-schema@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" + integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== + +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== + json5@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" @@ -2923,11 +3227,26 @@ jsonfile@^2.1.0: optionalDependencies: graceful-fs "^4.1.6" +jsonpath-plus@^0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/jsonpath-plus/-/jsonpath-plus-0.19.0.tgz#b901e57607055933dc9a8bef0cc25160ee9dd64c" + integrity sha512-GSVwsrzW9LsA5lzsqe4CkuZ9wp+kxBb2GwNniaWzI2YFn5Ig42rSW8ZxVpWXaAfakXNrx5pgY5AbQq7kzX29kg== + jsonschema@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2" integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw== +jsprim@^1.2.2: + version "1.4.2" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb" + integrity sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw== + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.4.0" + verror "1.10.0" + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -3123,6 +3442,18 @@ mime-db@1.48.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mime-types@~2.1.24: version "2.1.31" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" @@ -3157,11 +3488,40 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" +minimatch@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + minimist@^1.2.0, minimist@^1.2.5: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== +minipass@^3.0.0: + version "3.3.6" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" + integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== + dependencies: + yallist "^4.0.0" + +minipass@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-4.0.0.tgz#7cebb0f9fa7d56f0c5b17853cbe28838a8dbbd3b" + integrity sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw== + dependencies: + yallist "^4.0.0" + +minizlib@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" + integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mississippi@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022" @@ -3193,6 +3553,11 @@ mkdirp@^0.5.1, mkdirp@^0.5.3: dependencies: minimist "^1.2.5" +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + monaco-editor-webpack-plugin@1.9.0: version "1.9.0" resolved "https://registry.yarnpkg.com/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-1.9.0.tgz#5b547281b9f404057dc5d8c5722390df9ac90be6" @@ -3332,6 +3697,11 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -3346,6 +3716,11 @@ object-copy@^0.1.0: define-property "^0.2.5" kind-of "^3.0.3" +object-hash@^2.0.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5" + integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw== + object-keys@^1.0.12, object-keys@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" @@ -3375,6 +3750,11 @@ object.pick@^1.3.0: dependencies: isobject "^3.0.1" +oidc-token-hash@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/oidc-token-hash/-/oidc-token-hash-5.0.1.tgz#ae6beec3ec20f0fd885e5400d175191d6e2f10c6" + integrity sha512-EvoOtz6FIEBzE+9q253HsLCVRiK/0doEJ2HCvvqMQb3dHZrP3WlJKYtJ55CRTw4jmYomzH4wkPuCj/I3ZvpKxQ== + on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" @@ -3396,6 +3776,16 @@ onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" +openid-client@^5.3.0: + version "5.3.1" + resolved "https://registry.yarnpkg.com/openid-client/-/openid-client-5.3.1.tgz#69a5fa7d2b5ad479032f576852d40b4d4435488a" + integrity sha512-RLfehQiHch9N6tRWNx68cicf3b1WR0x74bJWHRc25uYIbSRwjxYcTFaRnzbbpls5jroLAaB/bFIodTgA5LJMvw== + dependencies: + jose "^4.10.0" + lru-cache "^6.0.0" + object-hash "^2.0.1" + oidc-token-hash "^5.0.1" + os-browserify@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27" @@ -3517,7 +3907,7 @@ path-key@^3.0.0, path-key@^3.1.0: resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== -path-parse@^1.0.6: +path-parse@^1.0.6, path-parse@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== @@ -3538,6 +3928,11 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== + picomatch@^2.0.4, picomatch@^2.2.1: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" @@ -3657,6 +4052,11 @@ prr@~1.0.1: resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= +psl@^1.1.28: + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== + public-encrypt@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/public-encrypt/-/public-encrypt-4.0.3.tgz#4fcc9d77a07e48ba7527e7cbe0de33d0701331e0" @@ -3704,7 +4104,7 @@ punycode@^1.2.4: resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= -punycode@^2.1.0: +punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== @@ -3714,6 +4114,11 @@ qs@6.7.0: resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== +qs@~6.5.2: + version "6.5.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" + integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== + querystring-es3@^0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73" @@ -3792,6 +4197,13 @@ readdirp@~3.5.0: dependencies: picomatch "^2.2.1" +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw== + dependencies: + resolve "^1.1.6" + rechoir@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.0.tgz#32650fd52c21ab252aa5d65b19310441c7e03aca" @@ -3870,6 +4282,32 @@ repeat-string@^1.6.1: resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= +request@^2.88.0: + version "2.88.2" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" + integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.5.0" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -3887,6 +4325,15 @@ resolve-url@^0.2.1: resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= +resolve@^1.1.6: + version "1.22.1" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" + integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== + dependencies: + is-core-module "^2.9.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^1.14.2, resolve@^1.9.0: version "1.20.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" @@ -3900,6 +4347,11 @@ ret@~0.1.10: resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== +rfc4648@^1.3.0: + version "1.5.2" + resolved "https://registry.yarnpkg.com/rfc4648/-/rfc4648-1.5.2.tgz#cf5dac417dd83e7f4debf52e3797a723c1373383" + integrity sha512-tLOizhR6YGovrEBLatX1sdcuhoSCXddw3mqNVAcKxGJ+J0hFeJ+SjeWCv5UPA/WU3YzWPPuCVYgXBKZUPGpKtg== + rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.3: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" @@ -3907,6 +4359,13 @@ rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.3: dependencies: glob "^7.1.3" +rimraf@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c" @@ -3939,7 +4398,7 @@ safe-regex@^1.1.0: dependencies: ret "~0.1.10" -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.1.0: +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -4081,6 +4540,15 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== +shelljs@^0.8.5: + version "0.8.5" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c" + integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow== + dependencies: + glob "^7.0.0" + interpret "^1.0.0" + rechoir "^0.6.2" + signal-exit@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" @@ -4167,6 +4635,21 @@ split-string@^3.0.1, split-string@^3.0.2: dependencies: extend-shallow "^3.0.0" +sshpk@^1.7.0: + version "1.17.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" + integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + ssri@^6.0.1: version "6.0.2" resolved "https://registry.yarnpkg.com/ssri/-/ssri-6.0.2.tgz#157939134f20464e7301ddba3e90ffa8f7728ac5" @@ -4195,6 +4678,11 @@ stream-browserify@^2.0.1: inherits "~2.0.1" readable-stream "^2.0.2" +stream-buffers@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/stream-buffers/-/stream-buffers-3.0.2.tgz#5249005a8d5c2d00b3a32e6e0a6ea209dc4f3521" + integrity sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ== + stream-each@^1.1.0: version "1.2.3" resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae" @@ -4267,11 +4755,28 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + tapable@^1.0.0, tapable@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== +tar@^6.1.11: + version "6.1.13" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.13.tgz#46e22529000f612180601a6fe0680e7da508847b" + integrity sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^4.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + tdigest@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021" @@ -4318,6 +4823,20 @@ timers-browserify@^2.0.4: dependencies: setimmediate "^1.0.4" +tmp-promise@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/tmp-promise/-/tmp-promise-3.0.3.tgz#60a1a1cc98c988674fcbfd23b6e3367bdeac4ce7" + integrity sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ== + dependencies: + tmp "^0.2.0" + +tmp@^0.2.0: + version "0.2.1" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" + integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== + dependencies: + rimraf "^3.0.0" + to-arraybuffer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43" @@ -4365,16 +4884,41 @@ toidentifier@1.0.0: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== +tslib@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e" + integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA== + tty-browserify@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6" integrity sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY= +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== + type-is@~1.6.17, type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" @@ -4388,6 +4932,11 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= +underscore@^1.13.6: + version "1.13.6" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441" + integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A== + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -4502,6 +5051,11 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@^3.3.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" @@ -4517,6 +5071,15 @@ vary@~1.1.2: resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + vm-browserify@^1.0.1: version "1.1.2" resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" @@ -4666,6 +5229,11 @@ ws@^5.2.0: dependencies: async-limiter "~1.0.0" +ws@^7.3.1: + version "7.5.9" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" + integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== + xtend@^4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From 65a3f43ec64d800f19e6b351425041a14aacea65 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 21 Dec 2022 12:59:46 -0700 Subject: [PATCH 104/123] Remove some unused variables and imports --- backend/langs.js | 3 +-- backend/sandbox.bash | 31 ++++++++++++++++--------------- backend/sandbox.js | 6 ------ backend/test-runner.js | 4 ++-- backend/util.js | 1 - 5 files changed, 19 insertions(+), 26 deletions(-) diff --git a/backend/langs.js b/backend/langs.js index c18af5c..8a0594e 100644 --- a/backend/langs.js +++ b/backend/langs.js @@ -1,5 +1,4 @@ -import fsOrig, { promises as fs } from "fs"; -import path from "path"; +import fsOrig from "fs"; import debounce from "debounce"; diff --git a/backend/sandbox.bash b/backend/sandbox.bash index 8c6f6ec..6bc61a3 100644 --- a/backend/sandbox.bash +++ b/backend/sandbox.bash @@ -1,17 +1,18 @@ +#!/usr/bin/env bash # This script is sourced by Bash within 'make sandbox'. if [[ -z "$L" ]]; then - echo 'environment variable unset: $L' >&2 + echo "environment variable unset: \$L" >&2 exit 1 fi if [[ -z "$LANG_CONFIG" ]]; then - echo 'environment variable unset: $LANG_CONFIG' >&2 + echo "environment variable unset: \$LANG_CONFIG" >&2 exit 1 fi function get { - jq -r ".$1" <<< "${LANG_CONFIG}" + jq -r ".$1" <<<"${LANG_CONFIG}" } function has { @@ -24,21 +25,21 @@ function riju-exec { function daemon { if has daemon; then - echo "$(get daemon)" + get daemon riju-exec "$(get daemon)" fi } function setup { if has setup; then - echo "$(get setup)" + get setup riju-exec "$(get setup)" fi } function repl { if has repl; then - echo "$(get repl)" + get repl riju-exec "$(get repl)" fi } @@ -47,22 +48,22 @@ function main { if get main | grep -q /; then mkdir -p "$(dirname "$(get main)")" fi - : > "$(get main)" - has prefix && get prefix >> "$(get main)" - get template >> "$(get main)" - has suffix && get suffix >> "$(get main)" + : >"$(get main)" + has prefix && get prefix >>"$(get main)" + get template >>"$(get main)" + has suffix && get suffix >>"$(get main)" } function compile { if has compile; then - echo "$(get compile)" + get compile riju-exec "$(get compile)" fi } function run-only { if has run; then - echo "$(get run)" + get run riju-exec "$(get run)" fi } @@ -73,18 +74,18 @@ function run { function format { if has format; then - echo "$(get format.run)" + get format.run riju-exec "( $(get format.run) ) < $(get main)" fi } function lsp { if has lsp.setup; then - echo "$(get lsp.setup)" + get lsp.setup riju-exec "$(get lsp.setup)" fi if has lsp; then - echo "$(get lsp.start)" + get lsp.start riju-exec "$(get lsp.start)" fi } diff --git a/backend/sandbox.js b/backend/sandbox.js index aef507c..fd3b693 100644 --- a/backend/sandbox.js +++ b/backend/sandbox.js @@ -6,11 +6,9 @@ import { readLangConfig } from "../lib/yaml.js"; import { bash, getUUID, - privilegedExec, privilegedPty, privilegedSession, quote, - run, } from "./util.js"; function die(msg) { @@ -18,10 +16,6 @@ function die(msg) { process.exit(1); } -function log(msg) { - console.log(msg); -} - async function main() { const sandboxScript = await fs.readFile("backend/sandbox.bash", "utf-8"); const lang = process.env.L; diff --git a/backend/test-runner.js b/backend/test-runner.js index 3ca350f..0eb617a 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -10,7 +10,7 @@ import { getTestHash } from "../lib/hash-test.js"; import * as api from "./api.js"; import { langsPromise } from "./langs.js"; import { shutdown } from "./shutdown.js"; -import { getUUID, run } from "./util.js"; +import { run } from "./util.js"; let langs = {}; @@ -525,7 +525,7 @@ const testTypes = { ensure: { pred: ({ ensure }) => (ensure ? true : false), }, - run: { pred: (config) => true }, + run: { pred: (_config) => true }, repl: { pred: ({ repl }) => (repl ? true : false), }, diff --git a/backend/util.js b/backend/util.js index 9113ab2..4637783 100644 --- a/backend/util.js +++ b/backend/util.js @@ -1,5 +1,4 @@ import { spawn } from "child_process"; -import os from "os"; import process from "process"; import * as Sentry from "@sentry/node"; From 7ea0e4f017bc2c71dc1ae9b671b1b305e971630d Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 21 Dec 2022 17:06:52 -0700 Subject: [PATCH 105/123] Spin up minio for object storage --- env.yaml.bash | 4 ++ k8s/namespaces.yaml | 6 ++ k8s/riju-docker-registry.yaml | 21 +++++++ k8s/riju-minio-rbac.yaml | 36 ++++++++++++ k8s/riju-minio.yaml | 102 ++++++++++++++++++++++++++++++++++ k8s/secrets.in.yaml | 29 ++++++++++ k8s/traefik-config.in.yaml | 7 +++ k8s/traefik.yaml | 2 + 8 files changed, 207 insertions(+) create mode 100644 k8s/riju-minio-rbac.yaml create mode 100644 k8s/riju-minio.yaml diff --git a/env.yaml.bash b/env.yaml.bash index 9bdd552..ac1a23e 100755 --- a/env.yaml.bash +++ b/env.yaml.bash @@ -21,4 +21,8 @@ metallb: registry: password: "${registry_password}" htpasswd: "$(htpasswd -nbB admin "${registry_password}")" + +minio: + accessKey: "$(head -c16 /dev/urandom | xxd -p)" + secretKey: "$(head -c16 /dev/urandom | xxd -p)" EOF diff --git a/k8s/namespaces.yaml b/k8s/namespaces.yaml index cc5f09d..86e4829 100644 --- a/k8s/namespaces.yaml +++ b/k8s/namespaces.yaml @@ -15,3 +15,9 @@ kind: Namespace apiVersion: v1 metadata: name: riju + +--- +kind: Namespace +apiVersion: v1 +metadata: + name: riju-user diff --git a/k8s/riju-docker-registry.yaml b/k8s/riju-docker-registry.yaml index 4e3df08..4eb007d 100644 --- a/k8s/riju-docker-registry.yaml +++ b/k8s/riju-docker-registry.yaml @@ -31,6 +31,27 @@ spec: containers: - name: registry image: "registry:2" + resources: {} + readinessProbe: + httpGet: + path: / + port: 5000 + scheme: HTTP + failureThreshold: 1 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: / + port: 5000 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 env: - name: REGISTRY_AUTH value: htpasswd diff --git a/k8s/riju-minio-rbac.yaml b/k8s/riju-minio-rbac.yaml new file mode 100644 index 0000000..a19d215 --- /dev/null +++ b/k8s/riju-minio-rbac.yaml @@ -0,0 +1,36 @@ +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + namespace: riju + name: minio + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: riju + name: minio +rules: + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - minio-keys + verbs: + - get + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: riju + name: minio +roleRef: + kind: Role + apiGroup: rbac.authorization.k8s.io + name: minio +subjects: + - kind: ServiceAccount + name: minio diff --git a/k8s/riju-minio.yaml b/k8s/riju-minio.yaml new file mode 100644 index 0000000..739a184 --- /dev/null +++ b/k8s/riju-minio.yaml @@ -0,0 +1,102 @@ +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + namespace: riju + name: minio +spec: + replicas: 1 + serviceName: minio + selector: + matchLabels: + app: minio + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 16Gi + storageClassName: openebs-hostpath + template: + metadata: + labels: + app: minio + spec: + serviceAccountName: minio + containers: + - name: minio + image: "minio/minio:RELEASE.2022-12-12T19-27-27Z" + resources: {} + readinessProbe: + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + failureThreshold: 1 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 2 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + args: + - "server" + - "/data" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-keys + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-keys + key: secret-key + ports: + - name: api + containerPort: 9000 + volumeMounts: + - name: data + mountPath: /data + +--- +kind: Service +apiVersion: v1 +metadata: + namespace: riju + name: minio +spec: + selector: + app: minio + ports: + - name: api + port: 80 + targetPort: 9000 + +--- +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 +metadata: + namespace: riju + name: minio +spec: + entryPoints: + - minio + routes: + - kind: Rule + match: "PathPrefix(`/`)" + services: + - namespace: riju + name: minio + port: 80 diff --git a/k8s/secrets.in.yaml b/k8s/secrets.in.yaml index 65118e8..44fe7db 100644 --- a/k8s/secrets.in.yaml +++ b/k8s/secrets.in.yaml @@ -34,3 +34,32 @@ stringData: } } } + +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: riju-user + name: registry-user-login +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + { + "auths": { + "localhost:30999": { + "username": "admin", + "password": "{{ .registry.password }}", + "auth": "{{ .registry.password | printf "admin:%s" | b64enc }}" + } + } + } + +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: riju + name: minio-keys +stringData: + access-key: "{{ .minio.accessKey }}" + secret-key: "{{ .minio.secretKey }}" diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml index 540346f..041bc8c 100644 --- a/k8s/traefik-config.in.yaml +++ b/k8s/traefik-config.in.yaml @@ -27,6 +27,13 @@ data: certResolver: riju domains: - main: k8s.riju.codes + minio: + address: ":32000" + http: + tls: + certResolver: riju + domains: + - main: k8s.riju.codes ping: entryPoint: "healthcheck" metrics: diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index d47954c..03bf0bd 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -132,6 +132,8 @@ spec: targetPort: 8443 - port: 31000 name: docker + - port: 32000 + name: minio --- kind: IngressClass From d6134d470c0be30acf1cadffe1a7b8568798fdb9 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 22 Dec 2022 12:08:24 -0700 Subject: [PATCH 106/123] v1 for server agent to be hosted on minio --- agent/go.mod | 5 + agent/go.sum | 2 + agent/main.go | 364 +++++++++++++++++++++++++++++++++++++++ backend/k8s.js | 123 +++++++++++++ k8s/riju-minio-rbac.yaml | 36 ---- k8s/riju-minio.yaml | 1 - k8s/secrets.in.yaml | 21 +++ 7 files changed, 515 insertions(+), 37 deletions(-) create mode 100644 agent/go.mod create mode 100644 agent/go.sum create mode 100644 agent/main.go create mode 100644 backend/k8s.js delete mode 100644 k8s/riju-minio-rbac.yaml diff --git a/agent/go.mod b/agent/go.mod new file mode 100644 index 0000000..8d3a29b --- /dev/null +++ b/agent/go.mod @@ -0,0 +1,5 @@ +module github.com/radian-software/riju/agent + +go 1.18 + +require github.com/gorilla/websocket v1.5.0 // indirect diff --git a/agent/go.sum b/agent/go.sum new file mode 100644 index 0000000..e5a03d4 --- /dev/null +++ b/agent/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/agent/main.go b/agent/main.go new file mode 100644 index 0000000..7b0dc44 --- /dev/null +++ b/agent/main.go @@ -0,0 +1,364 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "syscall" + "time" + + "github.com/gorilla/websocket" +) + +type clientMessage struct { + // "stdin" + Event string `json:"event"` + // contents of stdin + Data []byte `json:"data,omitempty"` +} + +type serverMessage struct { + // "start", "stdout", "stderr", "exit", "warn", "error" + Event string `json:"event"` + // contents of stdout/stderr + Data []byte `json:"data,omitempty"` + // error message + Text string `json:"text,omitempty"` + // exit status + ExitStatus *int `json:"exitStatus,omitempty"` +} + +const ( + pingPeriod = 15 * time.Second + pongWait = 10 * time.Second + writeDeadline = 1 * time.Second +) + +var upgrader = websocket.Upgrader{} + +func logWarn(err error) { + log.Println(err.Error()) +} + +func logWarnf(format string, arg ...interface{}) { + logWarn(fmt.Errorf(format, arg...)) +} + +func logError(err error) { + log.Println(err.Error()) +} + +func logErrorf(format string, arg ...interface{}) { + logError(fmt.Errorf(format, arg...)) +} + +func tryClose(obj io.Closer, objName string) { + err := obj.Close() + if err != nil { + logErrorf("error closing %s: %w", objName, err) + } +} + +func closeWs(ws *websocket.Conn) { + err := ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + if err != nil { + logErrorf("sending close message: %w", err) + } + tryClose(ws, "websocket") +} + +func send(ws *websocket.Conn, msg *serverMessage) { + data, err := json.Marshal(msg) + if err != nil { + logErrorf("marshaling message: %w", err) + closeWs(ws) + return + } + err = ws.WriteMessage(websocket.TextMessage, data) + if err != nil { + logErrorf("sending message: %w", err) + closeWs(ws) + return + } +} + +func fatal(ws *websocket.Conn, err error) { + send(ws, &serverMessage{ + Event: "fatal", + Text: err.Error(), + }) +} + +func fatalf(ws *websocket.Conn, format string, arg ...interface{}) { + fatal(ws, fmt.Errorf(format, arg...)) +} + +func warn(ws *websocket.Conn, err error) { + send(ws, &serverMessage{ + Event: "warn", + Text: err.Error(), + }) +} + +func warnf(ws *websocket.Conn, format string, arg ...interface{}) { + warn(ws, fmt.Errorf(format, arg...)) +} + +func handleClientMessages(ws *websocket.Conn, stdinChan chan<- []byte) { + // Close channel after we exit + defer close(stdinChan) + // Stop processing reads some time after we stop receiving + // timely responses to our pings. + ws.SetReadDeadline(time.Now().Add(pongWait)) + ws.SetPongHandler(func(string) error { + ws.SetReadDeadline(time.Now().Add(pongWait)) + return nil + }) + // Read data and dispatch appropriately. Return on timeout or + // error. Caller is responsible for cleanup. + for { + msgtype, data, err := ws.ReadMessage() + if err != nil { + fatalf(ws, "reading message: %w", err) + return + } + if msgtype != websocket.TextMessage { + fatalf(ws, "received non-text message type %d", msgtype) + return + } + msg := clientMessage{} + err = json.Unmarshal(data, &msg) + if err != nil { + fatalf(ws, "parsing json: %w", err) + return + } + switch msg.Event { + case "stdin": + stdinChan <- msg.Data + default: + logWarnf("received unknown event type %s", msg.Event) + } + } +} + +// https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go +func handler(w http.ResponseWriter, r *http.Request) { + // Upgrade http connection to websocket + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + logErrorf("upgrading connection: %w", err) + return + } + // Close websocket on error or when we exit + defer closeWs(ws) + // Parse request query parameters; do this after upgrading to + // websocket so that we can send errors back on the websocket + // which is easier for clients to parse + cmdline := r.URL.Query()["cmdline"] + if len(cmdline) == 0 { + fatalf(ws, "cmdline query parameter missing") + return + } + // Create pipes for communicating with subprocess + stdinRead, stdinWrite, err := os.Pipe() + if err != nil { + fatalf(ws, "creating stdin pipe: %w", err) + return + } + defer tryClose(stdinRead, "read end of stdin pipe") + defer tryClose(stdinWrite, "write end of stdin pipe") + stdoutRead, stdoutWrite, err := os.Pipe() + if err != nil { + fatalf(ws, "creating stdout pipe: %w", err) + return + } + defer tryClose(stdoutRead, "read end of stdout pipe") + defer tryClose(stdoutWrite, "write end of stdout pipe") + stderrRead, stderrWrite, err := os.Pipe() + if err != nil { + fatalf(ws, "creating stderr pipe: %w", err) + return + } + defer tryClose(stderrRead, "read end of stderr pipe") + defer tryClose(stderrWrite, "write end of stderr pipe") + // Spawn subprocess + proc, err := os.StartProcess(cmdline[0], cmdline, &os.ProcAttr{ + Files: []*os.File{stdinRead, stdoutWrite, stderrWrite}, + }) + if err != nil { + fatalf(ws, "spawning process: %w", err) + return + } + // Setup a way for other goroutines to report a fatal error, + // use increased capacity to avoid blockage with large number + // of write callsites + doneChan := make(chan struct{}, 10) + // Setup channels and variables to monitor process state + waitChan := make(chan struct{}, 1) + state := (*os.ProcessState)(nil) + // Monitor the process to see when it exits + go func() { + s, err := proc.Wait() + if err != nil { + fatalf(ws, "waiting for process to exit: %w", err) + } else { + state = s + } + waitChan <- struct{}{} + doneChan <- struct{}{} + }() + // Arrange to send information about the process exit status + // if we have obtained it by the time we return + defer func() { + if state != nil { + status := state.ExitCode() + send(ws, &serverMessage{ + Event: "exit", + ExitStatus: &status, + }) + } + }() + // Arrange for subprocess to be killed when we exit + defer func() { + // See if process has already exited or is about to + select { + case <-waitChan: + return + case <-time.NewTimer(500 * time.Millisecond).C: + // + } + // Try killing the process by closing stdin + tryClose(stdinWrite, "stdin to child") + select { + case <-waitChan: + return + case <-time.NewTimer(500 * time.Millisecond).C: + // + } + // Try killing the process with SIGTERM, SIGINT, then + // finally SIGKILL + for _, sig := range []os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL} { + err = proc.Signal(sig) + if err != nil { + logWarnf("sending %s to child: %w", sig.String(), err) + } + select { + case <-waitChan: + return + case <-time.NewTimer(500 * time.Millisecond).C: + // + } + } + // We are unable to kill the process + fatalf(ws, "unable to kill child") + }() + // Close our copies of pipe ends passed to subprocess + err = stdinRead.Close() + if err != nil { + fatalf(ws, "closing read end of stdin pipe from parent") + return + } + err = stdoutWrite.Close() + if err != nil { + fatalf(ws, "closing write end of stdout pipe from parent") + return + } + err = stderrWrite.Close() + if err != nil { + fatalf(ws, "closing write end of stderr pipe from parent") + return + } + // Handle received messages from client + stdinChan := make(chan []byte) + go func() { + handleClientMessages(ws, stdinChan) + doneChan <- struct{}{} + }() + go func() { + for data := range stdinChan { + _, err := stdinWrite.Write(data) + if err != nil { + warnf(ws, "writing to stdin: %w", err) + return + } + } + }() + // Send regular pings to ensure we get regular pongs to + // satisfy the read deadline on handleClientMessages + pingDoneChan := make(chan struct{}, 1) + defer func() { + pingDoneChan <- struct{}{} + }() + go func() { + ticker := time.NewTicker(pingPeriod) + defer ticker.Stop() + for { + select { + case <-ticker.C: + err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeDeadline)) + if err != nil { + logErrorf("sending ping: %w", err) + doneChan <- struct{}{} + return + } + case <-pingDoneChan: + return + } + } + }() + // Proxy stdout and stderr back to client + go func() { + for { + buf := make([]byte, 1024) + nr, err := stdoutRead.Read(buf) + if err != nil { + warnf(ws, "reading from stdout: %w", err) + return + } + if nr == 0 { + continue + } + data, err := json.Marshal(&serverMessage{ + Event: "stdout", + Data: buf[:nr], + }) + if err != nil { + fatalf(ws, "wrapping stdout in json: %w", err) + doneChan <- struct{}{} + return + } + ws.SetWriteDeadline(time.Now().Add(writeDeadline)) + err = ws.WriteMessage(websocket.TextMessage, data) + if err != nil { + fatalf(ws, "sending message: %w", err) + doneChan <- struct{}{} + return + } + } + }() + // Wait until either process is exited or a websocket + // operation fails + <-doneChan + // Process and websocket will be cleaned up after return + return +} + +func main() { + port := os.Getenv("RIJU_AGENT_PORT") + if port == "" { + port = "869" + } + host := os.Getenv("RIJU_AGENT_HOST") + if host == "" { + host = "0.0.0.0" + } + fmt.Printf("Listening on http://%s:%s\n", host, port) + err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), http.HandlerFunc(handler)) + if err != nil { + logError(err) + os.Exit(1) + } +} diff --git a/backend/k8s.js b/backend/k8s.js new file mode 100644 index 0000000..aeb1aa0 --- /dev/null +++ b/backend/k8s.js @@ -0,0 +1,123 @@ +import * as k8sClient from "@kubernetes/client-node"; + +const kubeconfig = new k8sClient.KubeConfig(); +kubeconfig.loadFromDefault(); + +const k8s = kubeconfig.makeApiClient(k8sClient.CoreV1Api); + +async function listUserSessions() { + return (await k8s.listNamespacedPod("riju-user")).body.items.map((pod) => ({ + podName: pod.metadata.name, + sessionID: pod.metadata.labels["riju.codes/user-session-id"], + })); +} + +async function createUserSession({ sessionID, langConfig, revisions }) { + await k8s.createNamespacedPod("riju-user", { + metadata: { + name: `riju-user-session-${sessionID}`, + labels: { + "riju.codes/user-session-id": sessionID, + }, + }, + spec: { + volumes: [ + { + name: "minio-config", + secret: { + secretName: "minio-user-login", + }, + }, + { + name: "riju-bin", + emptyDir: {}, + }, + ], + imagePullSecrets: [ + { + name: "registry-user-login", + }, + ], + initContainers: [ + { + name: "download", + image: "minio/mc:RELEASE.2022-12-13T00-23-28Z", + resources: {}, + args: [ + "sh", + "-c", + `mc cp riju/agent/${revisions.agent} /riju-bin/agent &&` + + `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify`, + ], + volumeMounts: [ + { + name: "minio-config", + mountPath: "/root/.mc", + readOnly: true, + }, + { + name: "riju-bin", + mountPath: "/riju-bin", + }, + ], + }, + ], + containers: [ + { + name: "session", + image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`, + resources: { + limits: { + cpu: "1000m", + memory: "4Gi", + }, + }, + startupProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 30, + initialDelaySeconds: 0, + periodSeconds: 1, + successThreshold: 1, + timeoutSeconds: 2, + }, + readinessProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 1, + initialDelaySeconds: 2, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 2, + }, + livenessProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 3, + initialDelaySeconds: 2, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 2, + }, + volumeMounts: [ + { + name: "riju-bin", + mountPath: "/riju-bin", + readOnly: true, + }, + ], + }, + ], + restartPolicy: "Never", + }, + }); +} diff --git a/k8s/riju-minio-rbac.yaml b/k8s/riju-minio-rbac.yaml deleted file mode 100644 index a19d215..0000000 --- a/k8s/riju-minio-rbac.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - namespace: riju - name: minio - ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: riju - name: minio -rules: - - apiGroups: - - "" - resources: - - secrets - resourceNames: - - minio-keys - verbs: - - get - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: riju - name: minio -roleRef: - kind: Role - apiGroup: rbac.authorization.k8s.io - name: minio -subjects: - - kind: ServiceAccount - name: minio diff --git a/k8s/riju-minio.yaml b/k8s/riju-minio.yaml index 739a184..486a937 100644 --- a/k8s/riju-minio.yaml +++ b/k8s/riju-minio.yaml @@ -24,7 +24,6 @@ spec: labels: app: minio spec: - serviceAccountName: minio containers: - name: minio image: "minio/minio:RELEASE.2022-12-12T19-27-27Z" diff --git a/k8s/secrets.in.yaml b/k8s/secrets.in.yaml index 44fe7db..dca6132 100644 --- a/k8s/secrets.in.yaml +++ b/k8s/secrets.in.yaml @@ -63,3 +63,24 @@ metadata: stringData: access-key: "{{ .minio.accessKey }}" secret-key: "{{ .minio.secretKey }}" + +--- +kind: Secret +apiVersion: v1 +metadata: + namespace: riju-user + name: minio-user-login +stringData: + config.json: | + { + "version": 10, + "aliases": { + "riju": { + "url": "http://minio.riju.svc", + "accessKey": "{{ .minio.accessKey }}", + "secretKey": "{{ .minio.secretKey }}", + "api": "s3v4", + "path": "auto" + } + } + } From 1477281cff72ab2078d8b78bf98575db15e34719 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 18:26:48 -0700 Subject: [PATCH 107/123] v2 for server agent --- .gitignore | 1 + agent/logging.go | 30 ++++ agent/main.go | 364 ++++++++++++++------------------------------- agent/process.go | 160 ++++++++++++++++++++ agent/websocket.go | 112 ++++++++++++++ 5 files changed, 412 insertions(+), 255 deletions(-) create mode 100644 agent/logging.go create mode 100644 agent/process.go create mode 100644 agent/websocket.go diff --git a/.gitignore b/.gitignore index 207808b..3934bba 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ env.yaml node_modules out sentinel.h +agent/agent diff --git a/agent/logging.go b/agent/logging.go new file mode 100644 index 0000000..7906905 --- /dev/null +++ b/agent/logging.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "io" + "log" +) + +func logWarn(err error) { + log.Println(err.Error()) +} + +func logWarnf(format string, arg ...interface{}) { + logWarn(fmt.Errorf(format, arg...)) +} + +func logError(err error) { + log.Println(err.Error()) +} + +func logErrorf(format string, arg ...interface{}) { + logError(fmt.Errorf(format, arg...)) +} + +func tryClose(obj io.Closer, objName string) { + err := obj.Close() + if err != nil { + logErrorf("error closing %s: %w", objName, err) + } +} diff --git a/agent/main.go b/agent/main.go index 7b0dc44..6d27742 100644 --- a/agent/main.go +++ b/agent/main.go @@ -3,11 +3,9 @@ package main import ( "encoding/json" "fmt" - "io" - "log" "net/http" "os" - "syscall" + "os/exec" "time" "github.com/gorilla/websocket" @@ -31,117 +29,64 @@ type serverMessage struct { ExitStatus *int `json:"exitStatus,omitempty"` } -const ( - pingPeriod = 15 * time.Second - pongWait = 10 * time.Second - writeDeadline = 1 * time.Second -) - var upgrader = websocket.Upgrader{} -func logWarn(err error) { - log.Println(err.Error()) +func closeWs(ms *ManagedWebsocket) { + ms.CloseChan <- struct{}{} } -func logWarnf(format string, arg ...interface{}) { - logWarn(fmt.Errorf(format, arg...)) -} - -func logError(err error) { - log.Println(err.Error()) -} - -func logErrorf(format string, arg ...interface{}) { - logError(fmt.Errorf(format, arg...)) -} - -func tryClose(obj io.Closer, objName string) { - err := obj.Close() - if err != nil { - logErrorf("error closing %s: %w", objName, err) - } -} - -func closeWs(ws *websocket.Conn) { - err := ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - logErrorf("sending close message: %w", err) - } - tryClose(ws, "websocket") -} - -func send(ws *websocket.Conn, msg *serverMessage) { +func send(ms *ManagedWebsocket, msg *serverMessage) { data, err := json.Marshal(msg) if err != nil { logErrorf("marshaling message: %w", err) - closeWs(ws) - return - } - err = ws.WriteMessage(websocket.TextMessage, data) - if err != nil { - logErrorf("sending message: %w", err) - closeWs(ws) + closeWs(ms) return } + ms.OutgoingChan <- data } -func fatal(ws *websocket.Conn, err error) { - send(ws, &serverMessage{ +func fatal(ms *ManagedWebsocket, err error) { + send(ms, &serverMessage{ Event: "fatal", Text: err.Error(), }) } -func fatalf(ws *websocket.Conn, format string, arg ...interface{}) { - fatal(ws, fmt.Errorf(format, arg...)) +func fatalf(ms *ManagedWebsocket, format string, arg ...interface{}) { + fatal(ms, fmt.Errorf(format, arg...)) } -func warn(ws *websocket.Conn, err error) { - send(ws, &serverMessage{ +func warn(ms *ManagedWebsocket, err error) { + send(ms, &serverMessage{ Event: "warn", Text: err.Error(), }) } -func warnf(ws *websocket.Conn, format string, arg ...interface{}) { - warn(ws, fmt.Errorf(format, arg...)) +func warnf(ms *ManagedWebsocket, format string, arg ...interface{}) { + warn(ms, fmt.Errorf(format, arg...)) } -func handleClientMessages(ws *websocket.Conn, stdinChan chan<- []byte) { - // Close channel after we exit - defer close(stdinChan) - // Stop processing reads some time after we stop receiving - // timely responses to our pings. - ws.SetReadDeadline(time.Now().Add(pongWait)) - ws.SetPongHandler(func(string) error { - ws.SetReadDeadline(time.Now().Add(pongWait)) - return nil - }) - // Read data and dispatch appropriately. Return on timeout or - // error. Caller is responsible for cleanup. - for { - msgtype, data, err := ws.ReadMessage() - if err != nil { - fatalf(ws, "reading message: %w", err) - return +func handleClientMessages(ms *ManagedWebsocket) <-chan []byte { + stdinChan := make(chan []byte, 16) + go func() { + defer close(stdinChan) + for data := range ms.IncomingChan { + msg := clientMessage{} + err := json.Unmarshal(data, &msg) + if err != nil { + fatalf(ms, "parsing json: %w", err) + return + } + switch msg.Event { + case "stdin": + stdinChan <- msg.Data + default: + logWarnf("received unknown event type %s", msg.Event) + } } - if msgtype != websocket.TextMessage { - fatalf(ws, "received non-text message type %d", msgtype) - return - } - msg := clientMessage{} - err = json.Unmarshal(data, &msg) - if err != nil { - fatalf(ws, "parsing json: %w", err) - return - } - switch msg.Event { - case "stdin": - stdinChan <- msg.Data - default: - logWarnf("received unknown event type %s", msg.Event) - } - } + }() + return stdinChan } // https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go @@ -152,197 +97,106 @@ func handler(w http.ResponseWriter, r *http.Request) { logErrorf("upgrading connection: %w", err) return } - // Close websocket on error or when we exit - defer closeWs(ws) + // Set up channels to handle incoming and outgoing websocket + // messages more conveniently, and also to handle closing the + // websocket on error or when we ask. + ms := &ManagedWebsocket{ + Socket: ws, + + MessageType: websocket.TextMessage, + PingPeriod: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + ms.Init() + // Ensure that websocket will be closed eventually when we + // exit. + defer closeWs(ms) // Parse request query parameters; do this after upgrading to // websocket so that we can send errors back on the websocket // which is easier for clients to parse cmdline := r.URL.Query()["cmdline"] if len(cmdline) == 0 { - fatalf(ws, "cmdline query parameter missing") + fatalf(ms, "cmdline query parameter missing") return } - // Create pipes for communicating with subprocess - stdinRead, stdinWrite, err := os.Pipe() + binary, err := exec.LookPath(cmdline[0]) if err != nil { - fatalf(ws, "creating stdin pipe: %w", err) + fatalf(ms, "searching for executable: %w", err) return } - defer tryClose(stdinRead, "read end of stdin pipe") - defer tryClose(stdinWrite, "write end of stdin pipe") - stdoutRead, stdoutWrite, err := os.Pipe() - if err != nil { - fatalf(ws, "creating stdout pipe: %w", err) - return - } - defer tryClose(stdoutRead, "read end of stdout pipe") - defer tryClose(stdoutWrite, "write end of stdout pipe") - stderrRead, stderrWrite, err := os.Pipe() - if err != nil { - fatalf(ws, "creating stderr pipe: %w", err) - return - } - defer tryClose(stderrRead, "read end of stderr pipe") - defer tryClose(stderrWrite, "write end of stderr pipe") // Spawn subprocess - proc, err := os.StartProcess(cmdline[0], cmdline, &os.ProcAttr{ - Files: []*os.File{stdinRead, stdoutWrite, stderrWrite}, - }) + mp, err := NewManagedProcess(binary, cmdline, nil) if err != nil { - fatalf(ws, "spawning process: %w", err) + fatalf(ms, "spawning process: %w", err) return } - // Setup a way for other goroutines to report a fatal error, - // use increased capacity to avoid blockage with large number - // of write callsites - doneChan := make(chan struct{}, 10) - // Setup channels and variables to monitor process state - waitChan := make(chan struct{}, 1) - state := (*os.ProcessState)(nil) - // Monitor the process to see when it exits - go func() { - s, err := proc.Wait() - if err != nil { - fatalf(ws, "waiting for process to exit: %w", err) - } else { - state = s - } - waitChan <- struct{}{} - doneChan <- struct{}{} - }() - // Arrange to send information about the process exit status - // if we have obtained it by the time we return + // Ensure eventual process termination defer func() { - if state != nil { - status := state.ExitCode() - send(ws, &serverMessage{ - Event: "exit", - ExitStatus: &status, - }) - } + mp.CloseChan <- struct{}{} }() - // Arrange for subprocess to be killed when we exit - defer func() { - // See if process has already exited or is about to - select { - case <-waitChan: - return - case <-time.NewTimer(500 * time.Millisecond).C: - // - } - // Try killing the process by closing stdin - tryClose(stdinWrite, "stdin to child") - select { - case <-waitChan: - return - case <-time.NewTimer(500 * time.Millisecond).C: - // - } - // Try killing the process with SIGTERM, SIGINT, then - // finally SIGKILL - for _, sig := range []os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL} { - err = proc.Signal(sig) - if err != nil { - logWarnf("sending %s to child: %w", sig.String(), err) - } - select { - case <-waitChan: - return - case <-time.NewTimer(500 * time.Millisecond).C: - // - } - } - // We are unable to kill the process - fatalf(ws, "unable to kill child") - }() - // Close our copies of pipe ends passed to subprocess - err = stdinRead.Close() - if err != nil { - fatalf(ws, "closing read end of stdin pipe from parent") - return - } - err = stdoutWrite.Close() - if err != nil { - fatalf(ws, "closing write end of stdout pipe from parent") - return - } - err = stderrWrite.Close() - if err != nil { - fatalf(ws, "closing write end of stderr pipe from parent") - return - } // Handle received messages from client - stdinChan := make(chan []byte) go func() { - handleClientMessages(ws, stdinChan) - doneChan <- struct{}{} - }() - go func() { - for data := range stdinChan { - _, err := stdinWrite.Write(data) + for data := range ms.IncomingChan { + msg := clientMessage{} + err := json.Unmarshal(data, &msg) if err != nil { - warnf(ws, "writing to stdin: %w", err) - return - } - } - }() - // Send regular pings to ensure we get regular pongs to - // satisfy the read deadline on handleClientMessages - pingDoneChan := make(chan struct{}, 1) - defer func() { - pingDoneChan <- struct{}{} - }() - go func() { - ticker := time.NewTicker(pingPeriod) - defer ticker.Stop() - for { - select { - case <-ticker.C: - err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeDeadline)) - if err != nil { - logErrorf("sending ping: %w", err) - doneChan <- struct{}{} - return - } - case <-pingDoneChan: - return - } - } - }() - // Proxy stdout and stderr back to client - go func() { - for { - buf := make([]byte, 1024) - nr, err := stdoutRead.Read(buf) - if err != nil { - warnf(ws, "reading from stdout: %w", err) - return - } - if nr == 0 { + warnf(ms, "parsing json: %w", err) continue } - data, err := json.Marshal(&serverMessage{ - Event: "stdout", - Data: buf[:nr], - }) - if err != nil { - fatalf(ws, "wrapping stdout in json: %w", err) - doneChan <- struct{}{} - return - } - ws.SetWriteDeadline(time.Now().Add(writeDeadline)) - err = ws.WriteMessage(websocket.TextMessage, data) - if err != nil { - fatalf(ws, "sending message: %w", err) - doneChan <- struct{}{} - return + switch msg.Event { + case "stdin": + mp.StdinChan <- msg.Data + default: + logWarnf("received unknown event type %s", msg.Event) } } }() - // Wait until either process is exited or a websocket - // operation fails - <-doneChan - // Process and websocket will be cleaned up after return + // Proxy stdout and stderr from subprocess + go func() { + for data := range mp.StdoutChan { + msg, err := json.Marshal(&serverMessage{ + Event: "stdout", + Data: data, + }) + if err != nil { + warnf(ms, "wrapping stdout in json: %w", err) + return + } + ms.OutgoingChan <- msg + } + }() + go func() { + for data := range mp.StderrChan { + msg, err := json.Marshal(&serverMessage{ + Event: "stderr", + Data: data, + }) + if err != nil { + warnf(ms, "wrapping stderr in json: %w", err) + return + } + ms.OutgoingChan <- msg + } + }() + // Send info about process exit status + exitChan2 := make(chan struct{}, 16) + go func() { + for status := range mp.ExitChan { + exitChan2 <- struct{}{} + code := status.ExitCode() + send(ms, &serverMessage{ + Event: "exit", + ExitStatus: &code, + }) + } + }() + // Wait until one of subprocess or websocket exits. The other + // one will be cleaned up on return. + select { + case <-exitChan2: + case <-ms.ClosedChan: + } return } diff --git a/agent/process.go b/agent/process.go new file mode 100644 index 0000000..ca20c07 --- /dev/null +++ b/agent/process.go @@ -0,0 +1,160 @@ +package main + +import ( + "fmt" + "os" + "syscall" + "time" +) + +type managedProcess struct { + proc *os.Process + + stdinRead *os.File + stdinWrite *os.File + stdoutRead *os.File + stdoutWrite *os.File + stderrRead *os.File + stderrWrite *os.File + + internalExitChan chan struct{} + + StdinChan chan []byte + StdoutChan chan []byte + StderrChan chan []byte + ExitChan chan *os.ProcessState + CloseChan chan struct{} +} + +func NewManagedProcess(name string, argv []string, attr *os.ProcAttr) (*managedProcess, error) { + mp := &managedProcess{ + StdinChan: make(chan []byte, 16), + StdoutChan: make(chan []byte, 16), + StderrChan: make(chan []byte, 16), + CloseChan: make(chan struct{}, 16), + } + done := false + go mp.handleClose() + defer func() { + if !done { + mp.CloseChan <- struct{}{} + } + }() + var err error + mp.stdinRead, mp.stdinWrite, err = os.Pipe() + if err != nil { + return mp, fmt.Errorf("creating stdin pipe: %w", err) + } + mp.stdoutRead, mp.stdoutWrite, err = os.Pipe() + if err != nil { + return mp, fmt.Errorf("creating stdout pipe: %w", err) + } + mp.stderrRead, mp.stderrWrite, err = os.Pipe() + if err != nil { + return mp, fmt.Errorf("creating stderr pipe: %w", err) + } + newAttr := &os.ProcAttr{} + if attr != nil { + *newAttr = *attr + } + if len(newAttr.Files) < 3 { + newAttr.Files = append(newAttr.Files, make([]*os.File, 3-len(newAttr.Files))...) + newAttr.Files[0] = mp.stdinRead + newAttr.Files[1] = mp.stdoutWrite + newAttr.Files[2] = mp.stderrWrite + } + mp.proc, err = os.StartProcess(name, argv, newAttr) + if err != nil { + return mp, fmt.Errorf("spawning process: %w", err) + } + go mp.handleWait() + go mp.handleInput(mp.StdinChan, mp.stdinWrite, "stdin") + go mp.handleOutput(mp.StdoutChan, mp.stdoutRead, "stdout") + go mp.handleOutput(mp.StderrChan, mp.stderrRead, "stderr") + done = true + return mp, nil +} + +func (mp *managedProcess) handleInput(ch chan []byte, f *os.File, name string) { + for data := range ch { + nw, err := f.Write(data) + if err != nil { + logWarnf("writing to %s: got error after %d of %d byte(s): %w", name, nw, len(data), err) + return + } + } +} + +func (mp *managedProcess) handleOutput(ch chan []byte, f *os.File, name string) { + for { + buf := make([]byte, 1024) + nr, err := f.Read(buf) + if err != nil { + logWarnf("reading from %s: got error after %d byte(s): %w", name, nr, err) + return + } + if nr == 0 { + continue + } + ch <- buf[:nr] + } +} + +func (mp *managedProcess) handleWait() { + s, err := mp.proc.Wait() + if err != nil { + logErrorf("waiting on process: %w", err) + } + mp.internalExitChan <- struct{}{} + mp.ExitChan <- s +} + +func (mp *managedProcess) killProc() { + // See if process has already exited or is about to + select { + case <-mp.internalExitChan: + return + case <-time.NewTimer(500 * time.Millisecond).C: + // + } + // Try killing the process by closing stdin + mp.stdinWrite.Close() + select { + case <-mp.internalExitChan: + return + case <-time.NewTimer(500 * time.Millisecond).C: + // + } + // Try killing the process with SIGTERM, SIGINT, then + // finally SIGKILL + for _, sig := range []os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL} { + err := mp.proc.Signal(sig) + if err != nil { + logErrorf("sending %s to child: %w", sig.String(), err) + } + select { + case <-mp.internalExitChan: + return + case <-time.NewTimer(500 * time.Millisecond).C: + // + } + } + // We are unable to kill the process + logErrorf("unable to kill child process (pid %d)", mp.proc.Pid) +} + +func (mp *managedProcess) handleClose() { + <-mp.CloseChan + for _, p := range []*os.File{ + mp.stdinRead, mp.stdinWrite, + mp.stdoutRead, mp.stdoutWrite, + mp.stderrRead, mp.stderrWrite, + } { + if p != nil { + p.Close() + } + } + if mp.proc != nil { + // + } +} diff --git a/agent/websocket.go b/agent/websocket.go new file mode 100644 index 0000000..dbe816b --- /dev/null +++ b/agent/websocket.go @@ -0,0 +1,112 @@ +package main + +import ( + "io" + "time" + + "github.com/gorilla/websocket" +) + +type ManagedWebsocket struct { + Socket *websocket.Conn + + MessageType int + PingPeriod time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + + IncomingChan chan []byte + OutgoingChan chan []byte + CloseChan chan struct{} + ClosedChan chan struct{} +} + +func (m *ManagedWebsocket) handleIncoming() { + pongChan := make(chan struct{}, 16) + m.Socket.SetPongHandler(func(string) error { + pongChan <- struct{}{} + return nil + }) + msgChan := make(chan []byte, 16) + go func() { + defer close(msgChan) + for { + msgtype, data, err := m.Socket.ReadMessage() + if err != nil { + if err != io.EOF { + logErrorf("reading message: %w", err) + } + m.Socket.Close() + return + } + if msgtype != m.MessageType { + logWarnf("ignoring message of unexpected type %d", msgtype) + continue + } + msgChan <- data + + } + }() + for { + m.Socket.SetReadDeadline(time.Now().Add(m.ReadTimeout)) + var msgtype int + var msgdata []byte + select { + case <-pongChan: + msgtype = websocket.PongMessage + case data := <-msgChan: + msgtype = m.MessageType + msgdata = data + } + if msgtype != m.MessageType { + continue + } + m.OutgoingChan <- msgdata + } +} + +func (m *ManagedWebsocket) handleOutgoing() { + pingTicker := time.NewTicker(m.PingPeriod) + defer pingTicker.Stop() + defer func() { + m.ClosedChan <- struct{}{} + }() + for { + var msgtype int + var msgdata []byte + select { + case <-pingTicker.C: + msgtype = websocket.PingMessage + msgdata = []byte{} + case data := <-m.OutgoingChan: + msgtype = m.MessageType + msgdata = data + case <-m.CloseChan: + msgtype = websocket.CloseMessage + msgdata = websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") + } + wd := time.Now().Add(m.WriteTimeout) + m.Socket.SetWriteDeadline(wd) + err := m.Socket.WriteMessage(msgtype, msgdata) + if err != nil { + logErrorf("writing message: %w", err) + m.Socket.Close() + return + } + if msgtype == websocket.CloseMessage { + time.Sleep(wd.Sub(time.Now())) + m.Socket.Close() + return + } + } +} + +func (m *ManagedWebsocket) Init() { + m.IncomingChan = make(chan []byte, 16) + m.OutgoingChan = make(chan []byte, 16) + m.CloseChan = make(chan struct{}, 16) + m.ClosedChan = make(chan struct{}, 16) + + go m.handleIncoming() + go m.handleOutgoing() +} From 56c315106939156e0d6df9fcd5c27afb80b00ce8 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 18:31:48 -0700 Subject: [PATCH 108/123] Fix some deadlocks --- agent/process.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/agent/process.go b/agent/process.go index ca20c07..5739e9e 100644 --- a/agent/process.go +++ b/agent/process.go @@ -28,9 +28,12 @@ type managedProcess struct { func NewManagedProcess(name string, argv []string, attr *os.ProcAttr) (*managedProcess, error) { mp := &managedProcess{ + internalExitChan: make(chan struct{}, 16), + StdinChan: make(chan []byte, 16), StdoutChan: make(chan []byte, 16), StderrChan: make(chan []byte, 16), + ExitChan: make(chan *os.ProcessState, 16), CloseChan: make(chan struct{}, 16), } done := false @@ -77,9 +80,9 @@ func NewManagedProcess(name string, argv []string, attr *os.ProcAttr) (*managedP func (mp *managedProcess) handleInput(ch chan []byte, f *os.File, name string) { for data := range ch { - nw, err := f.Write(data) + _, err := f.Write(data) if err != nil { - logWarnf("writing to %s: got error after %d of %d byte(s): %w", name, nw, len(data), err) + // Likely stdin closed by subprocess, this is normal return } } @@ -90,7 +93,8 @@ func (mp *managedProcess) handleOutput(ch chan []byte, f *os.File, name string) buf := make([]byte, 1024) nr, err := f.Read(buf) if err != nil { - logWarnf("reading from %s: got error after %d byte(s): %w", name, nr, err) + // Likely stdout/stderr closed by subprocess, + // this is normal return } if nr == 0 { From 32f4730a9c422af003c22d99501a1567a3920717 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 18:55:20 -0700 Subject: [PATCH 109/123] Fix another thing --- agent/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agent/main.go b/agent/main.go index 6d27742..c860add 100644 --- a/agent/main.go +++ b/agent/main.go @@ -197,6 +197,9 @@ func handler(w http.ResponseWriter, r *http.Request) { case <-exitChan2: case <-ms.ClosedChan: } + // Wait a bit to send any pending messages before closing the + // connection. + time.Sleep(1 * time.Second) return } From c26d06a7455bf7388e5ad281f48cedc19fb0b1be Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 18:56:56 -0700 Subject: [PATCH 110/123] Get rid of some logs --- agent/websocket.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/agent/websocket.go b/agent/websocket.go index dbe816b..b897d8c 100644 --- a/agent/websocket.go +++ b/agent/websocket.go @@ -1,7 +1,6 @@ package main import ( - "io" "time" "github.com/gorilla/websocket" @@ -33,9 +32,6 @@ func (m *ManagedWebsocket) handleIncoming() { for { msgtype, data, err := m.Socket.ReadMessage() if err != nil { - if err != io.EOF { - logErrorf("reading message: %w", err) - } m.Socket.Close() return } @@ -89,7 +85,6 @@ func (m *ManagedWebsocket) handleOutgoing() { m.Socket.SetWriteDeadline(wd) err := m.Socket.WriteMessage(msgtype, msgdata) if err != nil { - logErrorf("writing message: %w", err) m.Socket.Close() return } From bddd8a43ad52128a57e5044ec9cc1dab9b330a03 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 19:02:12 -0700 Subject: [PATCH 111/123] Remove unused function --- agent/main.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/agent/main.go b/agent/main.go index c860add..3c5b61a 100644 --- a/agent/main.go +++ b/agent/main.go @@ -67,28 +67,6 @@ func warnf(ms *ManagedWebsocket, format string, arg ...interface{}) { warn(ms, fmt.Errorf(format, arg...)) } -func handleClientMessages(ms *ManagedWebsocket) <-chan []byte { - stdinChan := make(chan []byte, 16) - go func() { - defer close(stdinChan) - for data := range ms.IncomingChan { - msg := clientMessage{} - err := json.Unmarshal(data, &msg) - if err != nil { - fatalf(ms, "parsing json: %w", err) - return - } - switch msg.Event { - case "stdin": - stdinChan <- msg.Data - default: - logWarnf("received unknown event type %s", msg.Event) - } - } - }() - return stdinChan -} - // https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go func handler(w http.ResponseWriter, r *http.Request) { // Upgrade http connection to websocket From adf468b85cba0ee0732b61c39e78ca20767a225b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 19:12:21 -0700 Subject: [PATCH 112/123] This was pretty silly --- agent/websocket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/websocket.go b/agent/websocket.go index b897d8c..15f19f6 100644 --- a/agent/websocket.go +++ b/agent/websocket.go @@ -57,7 +57,7 @@ func (m *ManagedWebsocket) handleIncoming() { if msgtype != m.MessageType { continue } - m.OutgoingChan <- msgdata + m.IncomingChan <- msgdata } } From 400088a2ce7161038defb63dbec970cdcb7e0b1f Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 19:24:04 -0700 Subject: [PATCH 113/123] Add unique name generator --- package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/package.json b/package.json index 3f7841a..50e3408 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "semaphore": "^1.1.0", "strip-ansi": "^6.0.0", "style-loader": "^2.0.0", + "unique-names-generator": "^4.7.1", "uuid": "^8.3.2", "vscode-languageserver-protocol": "3.15.3", "webpack": "^4.44.2", diff --git a/yarn.lock b/yarn.lock index f07fd44..d494211 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4977,6 +4977,11 @@ unique-filename@^1.1.1: dependencies: unique-slug "^2.0.0" +unique-names-generator@^4.7.1: + version "4.7.1" + resolved "https://registry.yarnpkg.com/unique-names-generator/-/unique-names-generator-4.7.1.tgz#966407b12ba97f618928f77322cfac8c80df5597" + integrity sha512-lMx9dX+KRmG8sq6gulYYpKWZc9RlGsgBR6aoO8Qsm3qvkSJ+3rAymr+TnV8EDMrIrwuFJ4kruzMWM/OpYzPoow== + unique-slug@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/unique-slug/-/unique-slug-2.0.2.tgz#baabce91083fc64e945b0f3ad613e264f7cd4e6c" From db46e6dd4884a94eb2282d5043263a06b7075422 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 19:47:46 -0700 Subject: [PATCH 114/123] Starting work on a k8s sandbox implementation --- backend/k8s.js | 23 ++++++++----- backend/sandbox-k8s.js | 77 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 backend/sandbox-k8s.js diff --git a/backend/k8s.js b/backend/k8s.js index aeb1aa0..c35dbfb 100644 --- a/backend/k8s.js +++ b/backend/k8s.js @@ -5,15 +5,15 @@ kubeconfig.loadFromDefault(); const k8s = kubeconfig.makeApiClient(k8sClient.CoreV1Api); -async function listUserSessions() { +export async function listUserSessions() { return (await k8s.listNamespacedPod("riju-user")).body.items.map((pod) => ({ podName: pod.metadata.name, sessionID: pod.metadata.labels["riju.codes/user-session-id"], })); } -async function createUserSession({ sessionID, langConfig, revisions }) { - await k8s.createNamespacedPod("riju-user", { +export async function createUserSession({ sessionID, langConfig, revisions }) { + const { body: pod } = await k8s.createNamespacedPod("riju-user", { metadata: { name: `riju-user-session-${sessionID}`, labels: { @@ -43,16 +43,16 @@ async function createUserSession({ sessionID, langConfig, revisions }) { name: "download", image: "minio/mc:RELEASE.2022-12-13T00-23-28Z", resources: {}, + command: ["sh", "-c"], args: [ - "sh", - "-c", - `mc cp riju/agent/${revisions.agent} /riju-bin/agent &&` + - `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify`, + `cp -RT /mc /root/.mc &&` + + `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` + + `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`, ], volumeMounts: [ { name: "minio-config", - mountPath: "/root/.mc", + mountPath: "/mc", readOnly: true, }, { @@ -120,4 +120,11 @@ async function createUserSession({ sessionID, langConfig, revisions }) { restartPolicy: "Never", }, }); + console.log(pod); +} + +export async function deleteUserSessions(sessionsToDelete) { + for (const { podName } of sessionsToDelete) { + await k8s.deleteNamespacedPod(podName, "riju-user"); + } } diff --git a/backend/sandbox-k8s.js b/backend/sandbox-k8s.js new file mode 100644 index 0000000..db616a5 --- /dev/null +++ b/backend/sandbox-k8s.js @@ -0,0 +1,77 @@ +import { spawn } from "child_process"; +import { promises as fs } from "fs"; +import process from "process"; + +import { readLangConfig } from "../lib/yaml.js"; +import * as k8s from "./k8s.js"; +import { getUUID, quote } from "./util.js"; + +function die(msg) { + console.error(msg); + process.exit(1); +} + +async function main() { + const sandboxScript = await fs.readFile("backend/sandbox.bash", "utf-8"); + const lang = process.env.L; + if (!lang) { + die("environment variable unset: $L"); + } + const langConfig = await readLangConfig(lang); + console.log(`Checking for existing sessions`); + const existingSessions = await k8s.listUserSessions(); + if (existingSessions.length > 0) { + console.log(`Killing ${existingSessions.length} existing session(s)`); + await k8s.deleteUserSessions(existingSessions); + } + const sessionID = getUUID(); + console.log(`Starting session with UUID ${sessionID}`); + await k8s.createUserSession({ + sessionID, + langConfig, + revisions: { + agent: "20221228-023645-invisible-amaranth-sparrow", + ptyify: "20221228-023645-clean-white-gorilla", + }, + }); + // let buffer = ""; + // await new Promise((resolve) => { + // session.stdout.on("data", (data) => { + // buffer += data.toString(); + // let idx; + // while ((idx = buffer.indexOf("\n")) !== -1) { + // const line = buffer.slice(0, idx); + // buffer = buffer.slice(idx + 1); + // if (line === "riju: container ready") { + // resolve(); + // } else { + // console.error(line); + // } + // } + // }); + // }); + // const args = [].concat.apply( + // ["riju-pty", "-f"], + // privilegedPty( + // { uuid }, + // bash( + // `env L='${lang}' LANG_CONFIG=${quote( + // JSON.stringify(langConfig) + // )} bash --rcfile <(cat <<< ${quote(sandboxScript)})` + // ) + // ) + // ); + // const proc = spawn(args[0], args.slice(1), { + // stdio: "inherit", + // }); + // try { + // await new Promise((resolve, reject) => { + // proc.on("error", reject); + // proc.on("close", resolve); + // }); + // } finally { + // session.kill(); + // } +} + +main().catch(die); From ed0be93886a32bcb18f5da6162a6e24739458096 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Tue, 27 Dec 2022 19:55:09 -0700 Subject: [PATCH 115/123] Fix some bugs --- backend/k8s.js | 3 ++- k8s/secrets.in.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/k8s.js b/backend/k8s.js index c35dbfb..11d33a0 100644 --- a/backend/k8s.js +++ b/backend/k8s.js @@ -45,7 +45,7 @@ export async function createUserSession({ sessionID, langConfig, revisions }) { resources: {}, command: ["sh", "-c"], args: [ - `cp -RT /mc /root/.mc &&` + + `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` + `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` + `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`, ], @@ -67,6 +67,7 @@ export async function createUserSession({ sessionID, langConfig, revisions }) { name: "session", image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`, resources: { + requests: {}, limits: { cpu: "1000m", memory: "4Gi", diff --git a/k8s/secrets.in.yaml b/k8s/secrets.in.yaml index dca6132..19e074d 100644 --- a/k8s/secrets.in.yaml +++ b/k8s/secrets.in.yaml @@ -73,7 +73,7 @@ metadata: stringData: config.json: | { - "version": 10, + "version": "10", "aliases": { "riju": { "url": "http://minio.riju.svc", From 48bdc9abf8e331a83ef1d7cad0ec0385b2eff346 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 28 Dec 2022 19:52:59 -0700 Subject: [PATCH 116/123] Add/change various things --- agent/go.mod | 5 +- agent/go.sum | 2 + agent/main.go | 28 +++- backend/k8s.js | 282 ++++++++++++++++++++++++++--------------- backend/sandbox-k8s.js | 5 +- env.yaml.bash | 5 + k8s/riju-proxy.yaml | 88 +++++++++++++ k8s/secrets.in.yaml | 9 ++ 8 files changed, 319 insertions(+), 105 deletions(-) create mode 100644 k8s/riju-proxy.yaml diff --git a/agent/go.mod b/agent/go.mod index 8d3a29b..bccdbc0 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -2,4 +2,7 @@ module github.com/radian-software/riju/agent go 1.18 -require github.com/gorilla/websocket v1.5.0 // indirect +require ( + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/gorilla/websocket v1.5.0 // indirect +) diff --git a/agent/go.sum b/agent/go.sum index e5a03d4..3b8bc42 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -1,2 +1,4 @@ +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/agent/main.go b/agent/main.go index 3c5b61a..1eda23e 100644 --- a/agent/main.go +++ b/agent/main.go @@ -8,6 +8,7 @@ import ( "os/exec" "time" + "github.com/google/shlex" "github.com/gorilla/websocket" ) @@ -67,6 +68,25 @@ func warnf(ms *ManagedWebsocket, format string, arg ...interface{}) { warn(ms, fmt.Errorf(format, arg...)) } +func getCommandPrefix() []string { + prefix := os.Getenv("RIJU_AGENT_COMMAND_PREFIX") + if prefix == "" { + logErrorf("must specify RIJU_AGENT_COMMAND_PREFIX for security reasons") + os.Exit(1) + } + if prefix == "0" { + return []string{} + } + list, err := shlex.Split(prefix) + if err != nil { + logErrorf("parsing RIJU_AGENT_COMMAND_PREFIX: %w", err) + os.Exit(1) + } + return list +} + +var CommandPrefix = getCommandPrefix() + // https://github.com/gorilla/websocket/blob/76ecc29eff79f0cedf70c530605e486fc32131d1/examples/command/main.go func handler(w http.ResponseWriter, r *http.Request) { // Upgrade http connection to websocket @@ -98,6 +118,7 @@ func handler(w http.ResponseWriter, r *http.Request) { fatalf(ms, "cmdline query parameter missing") return } + cmdline = append(CommandPrefix, cmdline...) binary, err := exec.LookPath(cmdline[0]) if err != nil { fatalf(ms, "searching for executable: %w", err) @@ -191,7 +212,12 @@ func main() { host = "0.0.0.0" } fmt.Printf("Listening on http://%s:%s\n", host, port) - err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), http.HandlerFunc(handler)) + mux := http.NewServeMux() + mux.HandleFunc("/exec", handler) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + err := http.ListenAndServe(fmt.Sprintf("%s:%s", host, port), mux) if err != nil { logError(err) os.Exit(1) diff --git a/backend/k8s.js b/backend/k8s.js index 11d33a0..116e283 100644 --- a/backend/k8s.js +++ b/backend/k8s.js @@ -1,10 +1,54 @@ import * as k8sClient from "@kubernetes/client-node"; +import lodash from "lodash"; const kubeconfig = new k8sClient.KubeConfig(); kubeconfig.loadFromDefault(); const k8s = kubeconfig.makeApiClient(k8sClient.CoreV1Api); +export function watchPods() { + const callbacks = {}; + const pods = {}; + + // https://github.com/kubernetes-client/javascript/blob/1f76ee10c54e33a998abb4686488ccff4285366a/examples/typescript/informer/informer.ts + // + // The watch functionality seems to be wholly undocumented. Copy, + // paste, and pray. + const informer = k8sClient.makeInformer( + kubeconfig, + "/api/v1/namespaces/riju-user/pods", + () => k8s.listNamespacedPod("riju-user") + ); + + for (const event of ["add", "update", "delete"]) { + informer.on(event, (pod) => { + if (pod.metadata.name in callbacks) { + callbacks[pod.metadata.name](event, pod); + } + pods[pod.metadata.name] = pod; + if (event == "delete") { + delete callbacks[pod.metadata.name]; + delete pods[pod.metadata.name]; + } + }); + } + + informer.on("error", (err) => { + console.error(err); + setTimeout(() => informer.start(), 5000); + }); + informer.start(); + + return { + setCallback: (podName, callback) => { + callbacks[podName] = callback; + if (podName in pods) { + callback("add", pods[podName]); + } + }, + }; +} + export async function listUserSessions() { return (await k8s.listNamespacedPod("riju-user")).body.items.map((pod) => ({ podName: pod.metadata.name, @@ -12,116 +56,150 @@ export async function listUserSessions() { })); } -export async function createUserSession({ sessionID, langConfig, revisions }) { - const { body: pod } = await k8s.createNamespacedPod("riju-user", { - metadata: { - name: `riju-user-session-${sessionID}`, - labels: { - "riju.codes/user-session-id": sessionID, +export async function createUserSession({ + watcher, + sessionID, + langConfig, + revisions, +}) { + const pod = ( + await k8s.createNamespacedPod("riju-user", { + metadata: { + name: `riju-user-session-${sessionID}`, + labels: { + "riju.codes/user-session-id": sessionID, + }, }, - }, - spec: { - volumes: [ - { - name: "minio-config", - secret: { - secretName: "minio-user-login", - }, - }, - { - name: "riju-bin", - emptyDir: {}, - }, - ], - imagePullSecrets: [ - { - name: "registry-user-login", - }, - ], - initContainers: [ - { - name: "download", - image: "minio/mc:RELEASE.2022-12-13T00-23-28Z", - resources: {}, - command: ["sh", "-c"], - args: [ - `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` + - `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` + - `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`, - ], - volumeMounts: [ - { - name: "minio-config", - mountPath: "/mc", - readOnly: true, - }, - { - name: "riju-bin", - mountPath: "/riju-bin", - }, - ], - }, - ], - containers: [ - { - name: "session", - image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`, - resources: { - requests: {}, - limits: { - cpu: "1000m", - memory: "4Gi", + spec: { + volumes: [ + { + name: "minio-config", + secret: { + secretName: "minio-user-login", }, }, - startupProbe: { - httpGet: { - path: "/health", - port: 869, - scheme: "HTTP", - }, - failureThreshold: 30, - initialDelaySeconds: 0, - periodSeconds: 1, - successThreshold: 1, - timeoutSeconds: 2, + { + name: "riju-bin", + emptyDir: {}, }, - readinessProbe: { - httpGet: { - path: "/health", - port: 869, - scheme: "HTTP", - }, - failureThreshold: 1, - initialDelaySeconds: 2, - periodSeconds: 10, - successThreshold: 1, - timeoutSeconds: 2, + ], + imagePullSecrets: [ + { + name: "registry-user-login", }, - livenessProbe: { - httpGet: { - path: "/health", - port: 869, - scheme: "HTTP", - }, - failureThreshold: 3, - initialDelaySeconds: 2, - periodSeconds: 10, - successThreshold: 1, - timeoutSeconds: 2, + ], + initContainers: [ + { + name: "download", + image: "minio/mc:RELEASE.2022-12-13T00-23-28Z", + resources: {}, + command: ["sh", "-c"], + args: [ + `mkdir -p /root/.mc && cp -LT /mc/config.json /root/.mc/config.json &&` + + `mc cp riju/agent/${revisions.agent} /riju-bin/agent && chmod +x /riju-bin/agent &&` + + `mc cp riju/ptyify/${revisions.ptyify} /riju-bin/ptyify && chmod +x /riju-bin/ptyify`, + ], + volumeMounts: [ + { + name: "minio-config", + mountPath: "/mc", + readOnly: true, + }, + { + name: "riju-bin", + mountPath: "/riju-bin", + }, + ], }, - volumeMounts: [ - { - name: "riju-bin", - mountPath: "/riju-bin", - readOnly: true, + ], + containers: [ + { + name: "session", + image: `localhost:30999/riju-lang:${langConfig.id}-${revisions.langImage}`, + resources: { + requests: {}, + limits: { + cpu: "1000m", + memory: "4Gi", + }, }, - ], - }, - ], - restartPolicy: "Never", - }, + command: ["/riju-bin/agent"], + env: [ + { + name: "RIJU_AGENT_COMMAND_PREFIX", + value: "runuser -u riju --", + }, + ], + securityContext: { + runAsUser: 0, + }, + startupProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 30, + initialDelaySeconds: 0, + periodSeconds: 1, + successThreshold: 1, + timeoutSeconds: 2, + }, + readinessProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 1, + initialDelaySeconds: 2, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 2, + }, + livenessProbe: { + httpGet: { + path: "/health", + port: 869, + scheme: "HTTP", + }, + failureThreshold: 3, + initialDelaySeconds: 2, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 2, + }, + volumeMounts: [ + { + name: "riju-bin", + mountPath: "/riju-bin", + readOnly: true, + }, + ], + }, + ], + restartPolicy: "Never", + }, + }) + ).body; + const podIP = await new Promise((resolve, reject) => { + setTimeout(() => reject("timed out"), 5 * 60 * 1000); + watcher.setCallback(pod.metadata.name, (event, pod) => { + if (event == "delete") { + reject(new Error("pod was deleted")); + } else if (pod.status.phase === "Failed") { + reject(new Error("pod status became Failed")); + } else if ( + pod.status.podIP && + lodash.every(pod.status.containerStatuses, (status) => status.ready) + ) { + resolve(pod.status.podIP); + } else { + console.log(event, JSON.stringify(pod.status, null, 2)); + } + }); }); - console.log(pod); + return podIP; } export async function deleteUserSessions(sessionsToDelete) { diff --git a/backend/sandbox-k8s.js b/backend/sandbox-k8s.js index db616a5..def9a81 100644 --- a/backend/sandbox-k8s.js +++ b/backend/sandbox-k8s.js @@ -26,12 +26,15 @@ async function main() { } const sessionID = getUUID(); console.log(`Starting session with UUID ${sessionID}`); + const watcher = k8s.watchPods(); await k8s.createUserSession({ + watcher, sessionID, langConfig, revisions: { - agent: "20221228-023645-invisible-amaranth-sparrow", + agent: "20221229-002450-semantic-moccasin-albatross", ptyify: "20221228-023645-clean-white-gorilla", + langImage: "20221227-195753-forward-harlequin-wolverine", }, }); // let buffer = ""; diff --git a/env.yaml.bash b/env.yaml.bash index ac1a23e..ebd6c24 100755 --- a/env.yaml.bash +++ b/env.yaml.bash @@ -5,6 +5,7 @@ set -euo pipefail cd "$(dirname "$0")" registry_password="$(pwgen -s 20 1)" +proxy_password="$(pwgen -s 20 1)" cat < Date: Wed, 28 Dec 2022 20:26:21 -0700 Subject: [PATCH 117/123] Fix bugs --- k8s/riju-proxy.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index 88fcb8c..0142ef0 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -8,6 +8,7 @@ data: squid.conf: | cache deny all + acl riju_src src 127.0.0.1/32 acl riju_src src 10.244.0.0/16 http_access deny !riju_src @@ -20,7 +21,7 @@ data: acl riju_method method GET http_access deny !riju_method - auth_param basic program /usr/lib/squid/basic_ncsa_auth + auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwd auth_param basic children 5 startup=5 idle=1 auth_param basic realm Riju administrative proxy auth_param basic credentialsttl 24 hours From 652662dbdf1e5580bb68cffdd84270d102a45a1e Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 28 Dec 2022 20:29:17 -0700 Subject: [PATCH 118/123] Fix logging --- k8s/riju-proxy.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index 0142ef0..08bc319 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -34,6 +34,10 @@ data: http_port 3128 + # Avoid logging TCP healthchecks as errors + acl hasRequest has request + access_log daemon:/var/log/squid/access.log hasRequest + --- kind: Deployment apiVersion: apps/v1 From be76ff95d4764f30b5af06136fc0d4d3faf9c3df Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 28 Dec 2022 20:43:35 -0700 Subject: [PATCH 119/123] Expose squid via traefik (behind auth) --- k8s/riju-proxy.yaml | 31 +++++++++++++++++++++++++++++++ k8s/traefik-config.in.yaml | 7 +++++++ k8s/traefik.yaml | 2 ++ 3 files changed, 40 insertions(+) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index 08bc319..7e1543d 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -91,3 +91,34 @@ spec: - name: auth mountPath: /etc/squid/passwd subPath: htpasswd + +--- +kind: Service +apiVersion: v1 +metadata: + namespace: riju + name: riju-proxy +spec: + selector: + app: riju-proxy + ports: + - name: http + port: 3128 + targetPort: 3128 + +--- +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 +metadata: + namespace: riju + name: riju-proxy +spec: + entryPoints: + - proxy + routes: + - kind: Rule + match: "PathPrefix(`/`)" + services: + - namespace: riju + name: riju-proxy + port: 3128 diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml index 041bc8c..a4942fe 100644 --- a/k8s/traefik-config.in.yaml +++ b/k8s/traefik-config.in.yaml @@ -7,6 +7,13 @@ metadata: data: traefik.yaml: | entryPoints: + proxy: + address: ":3128" + http: + tls: + certResolver: riju + domains: + - main: k8s.riju.codes http: address: ":8000" https: diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index 03bf0bd..78a3f26 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -124,6 +124,8 @@ spec: selector: app: traefik ports: + - port: 3128 + name: squid - port: 80 name: http targetPort: 8000 From 4bc1ef4591f10e34aced4d3497aa918924845805 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Wed, 28 Dec 2022 23:52:37 -0700 Subject: [PATCH 120/123] Try it a different way (also doesn't work) This reverts commit be76ff95d4764f30b5af06136fc0d4d3faf9c3df. --- k8s/riju-proxy.yaml | 20 +++----------------- k8s/traefik-config.in.yaml | 7 ------- k8s/traefik.yaml | 4 ++-- 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index 7e1543d..af77258 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -98,27 +98,13 @@ apiVersion: v1 metadata: namespace: riju name: riju-proxy + annotations: + metallb.universe.tf/allow-shared-ip: main spec: + type: LoadBalancer selector: app: riju-proxy ports: - name: http port: 3128 targetPort: 3128 - ---- -kind: IngressRoute -apiVersion: traefik.containo.us/v1alpha1 -metadata: - namespace: riju - name: riju-proxy -spec: - entryPoints: - - proxy - routes: - - kind: Rule - match: "PathPrefix(`/`)" - services: - - namespace: riju - name: riju-proxy - port: 3128 diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml index a4942fe..041bc8c 100644 --- a/k8s/traefik-config.in.yaml +++ b/k8s/traefik-config.in.yaml @@ -7,13 +7,6 @@ metadata: data: traefik.yaml: | entryPoints: - proxy: - address: ":3128" - http: - tls: - certResolver: riju - domains: - - main: k8s.riju.codes http: address: ":8000" https: diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index 78a3f26..f3a8a0f 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -119,13 +119,13 @@ apiVersion: v1 metadata: namespace: traefik name: traefik + annotations: + metallb.universe.tf/allow-shared-ip: main spec: type: LoadBalancer selector: app: traefik ports: - - port: 3128 - name: squid - port: 80 name: http targetPort: 8000 From 7d7bb6d4d3d2dc5e28c8bfea1ea2d89492d7ad6b Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 29 Dec 2022 00:23:26 -0700 Subject: [PATCH 121/123] Just use nginx instead istg --- k8s/riju-proxy.yaml | 77 +++++++++++++++++--------------------- k8s/traefik-config.in.yaml | 7 ++++ k8s/traefik.yaml | 2 + 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index af77258..b2bc351 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -5,38 +5,18 @@ metadata: namespace: riju name: riju-proxy-config data: - squid.conf: | - cache deny all + default.conf: | + server { + resolver kube-dns.kube-system.svc.cluster.local; + listen 1869 default_server; - acl riju_src src 127.0.0.1/32 - acl riju_src src 10.244.0.0/16 - http_access deny !riju_src + auth_basic "Riju administrative proxy"; + auth_basic_user_file /etc/nginx/passwd; - acl riju_dst dst 10.244.0.0/16 - http_access deny !riju_dst - - acl riju_port port 869 - http_access deny !riju_port - - acl riju_method method GET - http_access deny !riju_method - - auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwd - auth_param basic children 5 startup=5 idle=1 - auth_param basic realm Riju administrative proxy - auth_param basic credentialsttl 24 hours - - acl riju_auth proxy_auth REQUIRED - http_access deny !riju_auth - - http_access allow all - http_access deny all - - http_port 3128 - - # Avoid logging TCP healthchecks as errors - acl hasRequest has request - access_log daemon:/var/log/squid/access.log hasRequest + location ~ { + proxy_pass http://$host:869; + } + } --- kind: Deployment @@ -63,11 +43,11 @@ spec: secretName: riju-proxy-auth containers: - name: nginx - image: "ubuntu/squid:5.2-22.04_beta" + image: "nginx:1.23" resources: {} readinessProbe: tcpSocket: - port: 3128 + port: 1869 failureThreshold: 1 initialDelaySeconds: 2 periodSeconds: 10 @@ -75,7 +55,7 @@ spec: timeoutSeconds: 2 livenessProbe: tcpSocket: - port: 3128 + port: 1869 failureThreshold: 3 initialDelaySeconds: 2 periodSeconds: 10 @@ -83,13 +63,12 @@ spec: timeoutSeconds: 2 ports: - name: http - containerPort: 3128 + containerPort: 1869 volumeMounts: - name: config - mountPath: /etc/squid/squid.conf - subPath: squid.conf + mountPath: /etc/nginx/conf.d - name: auth - mountPath: /etc/squid/passwd + mountPath: /etc/nginx/passwd subPath: htpasswd --- @@ -98,13 +77,27 @@ apiVersion: v1 metadata: namespace: riju name: riju-proxy - annotations: - metallb.universe.tf/allow-shared-ip: main spec: - type: LoadBalancer selector: app: riju-proxy ports: - name: http - port: 3128 - targetPort: 3128 + port: 1869 + targetPort: 1869 + +--- +kind: IngressRoute +apiVersion: traefik.containo.us/v1alpha1 +metadata: + namespace: riju + name: riju-proxy +spec: + entryPoints: + - proxy + routes: + - kind: Rule + match: "PathPrefix(`/`)" + services: + - namespace: riju + name: riju-proxy + port: 1869 diff --git a/k8s/traefik-config.in.yaml b/k8s/traefik-config.in.yaml index 041bc8c..52ff2e6 100644 --- a/k8s/traefik-config.in.yaml +++ b/k8s/traefik-config.in.yaml @@ -7,6 +7,13 @@ metadata: data: traefik.yaml: | entryPoints: + proxy: + address: ":1869" + http: + tls: + certResolver: riju + domains: + - main: k8s.riju.codes http: address: ":8000" https: diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml index f3a8a0f..0a4b749 100644 --- a/k8s/traefik.yaml +++ b/k8s/traefik.yaml @@ -132,6 +132,8 @@ spec: - port: 443 name: https targetPort: 8443 + - port: 1869 + name: proxy - port: 31000 name: docker - port: 32000 From a19af3ba28068acfbaee98b2aaf2dd1d8fb83fb2 Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 29 Dec 2022 00:33:36 -0700 Subject: [PATCH 122/123] More things --- k8s/riju-proxy.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index b2bc351..7cbf131 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -6,6 +6,8 @@ metadata: name: riju-proxy-config data: default.conf: | + underscores_in_headers on; + server { resolver kube-dns.kube-system.svc.cluster.local; listen 1869 default_server; @@ -13,8 +15,12 @@ data: auth_basic "Riju administrative proxy"; auth_basic_user_file /etc/nginx/passwd; - location ~ { - proxy_pass http://$host:869; + location ~ /(10\.[0-9]+\.[0-9]+\.[0-9]+)/(.*) { + proxy_pass http://$1:869/$2; + } + + location / { + return 404; } } From d370c5fbc85bf9f479adca7c532ec13c2b54199f Mon Sep 17 00:00:00 2001 From: Radon Rosborough Date: Thu, 29 Dec 2022 00:45:38 -0700 Subject: [PATCH 123/123] omfg it works --- k8s/riju-proxy.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/k8s/riju-proxy.yaml b/k8s/riju-proxy.yaml index 7cbf131..cbd0847 100644 --- a/k8s/riju-proxy.yaml +++ b/k8s/riju-proxy.yaml @@ -15,8 +15,16 @@ data: auth_basic "Riju administrative proxy"; auth_basic_user_file /etc/nginx/passwd; - location ~ /(10\.[0-9]+\.[0-9]+\.[0-9]+)/(.*) { - proxy_pass http://$1:869/$2; + location ~ /(10\.[0-9]+\.[0-9]+\.[0-9]+)/health { + proxy_pass http://$1:869/health; + } + + location ~ /(10\.[0-9]+\.[0-9]+\.[0-9]+)/exec { + proxy_pass http://$1:869/exec$is_args$args; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; } location / {