diff --git a/Makefile b/Makefile index 78bef06..0ef0fee 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ else SHELL_PORTS := endif -SHELL_ENV := -e Z -e CI -e TEST_PATIENCE -e TEST_CONCURRENCY +SHELL_ENV := -e Z -e CI -e TEST_PATIENCE -e TEST_CONCURRENCY -e TEST_TIMEOUT_SECS -e FATHOM_SITE_ID ifeq ($(I),lang) LANG_TAG := lang-$(L) @@ -202,6 +202,12 @@ lsp: # L= : Run LSP REPL for language or custom command line ### Fetch artifacts from registries +PUBLIC_DOCKER_REPO_PULL ?= public.ecr.aws/raxod502/riju + +sync-ubuntu: # Pull Riju Ubuntu image from public Docker registry + docker pull $(PUBLIC_DOCKER_REPO_PULL):ubuntu + docker tag $(PUBLIC_DOCKER_REPO_PULL):ubuntu riju:ubuntu + pull: # I= : Pull last published Riju image from Docker registry @: $${I} $${DOCKER_REPO} docker pull $(DOCKER_REPO):$(I) @@ -239,9 +245,11 @@ upload: # L= T= : Upload .deb to S3 deploy-config: # Generate deployment config file node tools/generate-deploy-config.js -deploy: deploy-config # Upload deployment config to S3 and update ASG instances +deploy-latest: # Upload deployment config to S3 and update ASG instances aws s3 cp $(BUILD)/config.json $(S3_CONFIG) +deploy: deploy-config deploy-latest # Shorthand for deploy-config followed by deploy-latest + ### Infrastructure packer: supervisor # Build and publish a new AMI diff --git a/backend/server.js b/backend/server.js index c1a6bc4..a7e4013 100644 --- a/backend/server.js +++ b/backend/server.js @@ -14,7 +14,7 @@ const host = process.env.HOST || "localhost"; const port = parseInt(process.env.PORT || "") || 6119; const tlsPort = parseInt(process.env.TLS_PORT || "") || 6120; const useTLS = process.env.TLS ? true : false; -const analyticsEnabled = process.env.ANALYTICS ? true : false; +const fathomSiteId = process.env.FATHOM_SITE_ID || ""; const app = express(); @@ -25,7 +25,7 @@ app.get("/", (_, res) => { if (Object.keys(langs).length > 0) { res.render(path.resolve("frontend/pages/index"), { langs, - analyticsEnabled, + fathomSiteId, }); } else { res @@ -59,7 +59,7 @@ app.get("/:lang", (req, res) => { } res.render(path.resolve("frontend/pages/app"), { config: langs[lang], - analyticsEnabled, + fathomSiteId, }); }); app.use("/css", express.static("frontend/styles")); diff --git a/backend/test-runner.js b/backend/test-runner.js index bd6f589..4aa5736 100644 --- a/backend/test-runner.js +++ b/backend/test-runner.js @@ -19,7 +19,7 @@ function parseIntOr(thing, def) { return Number.isNaN(num) ? def : num; } -const TIMEOUT = parseIntOr(process.env.TEST_TIMEOUT_SECS, 15); +const TIMEOUT = parseIntOr(process.env.TEST_TIMEOUT_SECS, 30); const PATIENCE = parseIntOr(process.env.TEST_PATIENCE, 1); const CONCURRENCY = parseIntOr(process.env.TEST_CONCURRENCY, 2); diff --git a/doc/build.md b/doc/build.md index d9e140e..277be55 100644 --- a/doc/build.md +++ b/doc/build.md @@ -4,6 +4,39 @@ Riju's build system is complex and takes some time to explain. Bear with me. (If you just want to add or modify a language, you can read the [tutorial](tutorial.md) instead.) +## Depgraph + +The high level interface to Riju's build system is a tool called +Depgraph, which knows about all the build artifacts and has advanced +mechanisms for determining which of them need to rebuild based on +content hashes. Normally you can just use Depgraph to build the +artifacts you need. However, in some cases you may want to interact +with the lower level for more precise operations. This is done via +Makefile. (Furthermore, there are a few one-off artifacts such as the +admin image which are not part of the main build system, which means +that they are managed directly by Makefile.) + +### Available build artifacts + + + +### Usage of Depgraph + +``` +$ dep --help +Usage: dep ... + +Options: + --list list available artifacts; ignore other arguments + --manual operate explicitly on manual artifacts + --hold-manual prefer local versions of manual artifacts + --all do not skip unneeded intermediate artifacts + --local-only do not fetch artifacts from remote registries + --publish publish artifacts to remote registries + --yes execute plan without confirmation + -h, --help display help for command +``` + To get a quick overview, run `make help`. ## Build artifacts diff --git a/doc/infrastructure.md b/doc/infrastructure.md index 24073c3..87f937e 100644 --- a/doc/infrastructure.md +++ b/doc/infrastructure.md @@ -3,13 +3,30 @@ You can host your own instance of Riju! This requires a bit of manual setup, but everything that *can* be automated, *has* been automated. +**Warning: AWS is expensive and you are responsible for your own +spending. If you would be perturbed by accidentally burning a few +hundred dollars on unexpected compute, you probably shouldn't follow +these instructions.** + ## Sign up for accounts * [AWS](https://aws.amazon.com/) -* [Docker Hub](https://hub.docker.com/) +* [CloudFlare](https://www.cloudflare.com/) +* [Fathom Analytics](https://usefathom.com/) (if you want analytics) * [GitHub](https://github.com/) +* [Namecheap](https://www.namecheap.com/) +* [PagerDuty](https://www.pagerduty.com/) (if you want alerts) ## Configure accounts +### GitHub + +Fork the Riju repository under your account. + +### PagerDuty + +Set up notification rules as desired. Configure the AWS CloudWatch +integration and obtain an integration URL. + ### AWS You need to generate an access key with sufficient permission to apply @@ -23,24 +40,35 @@ don't already know your way around IAM is: * Attach the "AdministratorAccess" policy * Copy and save the access key ID and secret access key -You also need to create an S3 bucket to store Terraform's state. Go to -[S3 in -us-west-1](https://s3.console.aws.amazon.com/s3/home?region=us-west-1#) -and create a new bucket called `riju-yourname-tf`. +You also need to create an S3 bucket to store Terraform state. [Go to +S3](https://s3.console.aws.amazon.com/s3/home?region=us-west-1), +select your favorite AWS region, and create a new bucket called +`riju-yourname-tf`. -### Docker Hub +Finally, if you don't have a personal SSH key, generate one with +`ssh-keygen`, and upload the public key (e.g. `~/.ssh/id_rsa.pub`) as +an [EC2 Key +Pair](https://us-west-1.console.aws.amazon.com/ec2/v2/home?region=us-west-1#KeyPairs:). +Remember the name you use for the key pair. -Create a new repository to use for Riju. +### Namecheap -### GitHub +Buy a domain name at which to host. -Fork the Riju repository under your account. +### CloudFlare + +Enter your domain name and go through the setup and DNS verification. +Update the nameserver settings on Namecheap's side, and enable all the +fun CloudFlare options you'd like. + +### Fathom Analytics + +Enter your domain name and get a site ID. ## Install dependencies * [Docker](https://www.docker.com/) * [Git](https://git-scm.com/) -* [SSH](https://www.openssh.com/) ## Set up Riju locally @@ -59,164 +87,168 @@ and refer to a [tmux cheatsheet](https://danielmiessler.com/study/tmux/) if you are unfamiliar with tmux usage. -## Configure your instance -### AWS +## Authenticate with AWS -Sign in locally to the AWS CLI by running +Run `aws configure` and enter your IAM access credentials and your +preferred AWS region. - $ aws configure +## Create local configuration (part 1 of 3) -and entering the access key that you generated on AWS. The default -region is unimportant, although you may want to set it to `us-west-1` -because this is where Riju is configured to be deployed and having -that be consistent with your default command-line environment may -reduce confusion. - -### Password - -You need an administrator password that will be used for `sudo` access -on the production instance of Riju. You can generate one using `pwgen --s 20 1`. - -### SSH keys - -You need two keys. One is used for administrator login on the -production instance, and the other is used to trigger deployments. You -can use your personal SSH key, if you already have one, for the admin -key. However, you should definitely generate a new key for -deployments. To generate an SSH key, use the `ssh-keygen` utility. - -Place both keys in `~/.ssh`. This directory is automatically mounted -into the admin shell at `/home/riju/.ssh`. - -### Additional configuration - -You also need to have: - -* The name of the Docker Hub repository that you created earlier (e.g. - `raxod502/riju`). -* The base name of the S3 bucket(s) for Riju that will be created in - your AWS account. The official buckets use prefix `riju`, but since - S3 buckets must be globally unique you should use `riju-yourname`. - -With configuration in hand, create a file `.env` in the Riju directory -with the following contents, adjusting the values to match your -configuration: +Create a file `.env` in the Riju directory with the following +contents, referring to the following sub-sections for how to fill in +the values properly: ``` +# Packer ADMIN_PASSWORD=50M9QDBtkQLV6zFAwhVg -ADMIN_SSH_PUBLIC_KEY_FILE=/home/riju/.ssh/id_rsa.pub -DEPLOY_SSH_PUBLIC_KEY_FILE=/home/riju/.ssh/id_rsa_riju_deploy.pub -DOCKER_REPO=raxod502/riju -S3_BUCKET=riju-yourname +FATHOM_SITE_ID= +SUPERVISOR_ACCESS_TOKEN=5ta2qzMFS417dG9gbfcMgjsbDnGMI4 ``` -## Create infrastructure +### ADMIN\_PASSWORD -Run `make env` in the admin shell to start a subshell with environment -variables set from `.env`. Your first step will be to create an -[AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) -for Riju. +This will be the `sudo` password for Riju server nodes. Generate one +randomly with `pwgen -s 20 1`. + +### FATHOM\_SITE\_ID + +This is the site ID from your Fathom Analytics account. If you don't +need analytics, just leave this unset. + +### SUPERVISOR\_ACCESS\_TOKEN + +This is a static shared secret used for the Riju server's supervisor +API. Generate one randomly with `pwgen -s 30 1`. + +## Build AMI + +You'll want to run `make env` to load in the new variables from +`.env`. Now run `make packer`. This will take up to 10 minutes to +build a timestamped AMI with a name like `riju-20210711223158`. + +## Create local configuration (part 2 of 3) + +Add to `.env` the following contents: ``` -$ cd packer -$ packer build config.json +# Terraform +AMI_NAME=riju-20210711223158 +AWS_REGION=us-west-1 +S3_BUCKET=yourname-riju +SSH_KEY_NAME=something ``` -This will take several minutes. Note the name of the AMI that is -printed, and add a corresponding line to `.env`: +### AMI\_NAME + +This is the AMI name from the Packer build. + +### AWS\_REGION + +This is the region in which most Terraform infrastructure will be +created. It should be the same as the default region you configured +for the AWS CLI. It doesn't have to be the same as the region in which +your Terraform state bucket is configured, although it simplifies +matters to keep them in the same region. + +The main utility of having this as an explicit environment variable is +that Terraform respects it and won't always ask you what region to +use. + +### S3\_BUCKET + +This is the name of the S3 bucket that will be used to store Riju +build artifacts (aside from Docker images). It needs to be globally +unique, so `yourname-riju` is a good choice. + +### SSH\_KEY\_NAME + +This is the name of the EC2 Key Pair you created in the AWS console. +You'll use it to connect to the development server. + +## Set up Terraform infrastructure + +Run `make env` again to load in the new variables from `.env`. + +Now run `terraform init` and fill in the appropriate region and bucket +name for the Terraform state bucket you created in the AWS console. + +At this point you can run `terraform apply` to create all the needed +infrastructure. Caution! At this point you probably want to go to the +EC2 console and stop the dev server. It is very expensive and will +rack up a few hundred dollars a month of compute. You should only have +it running when you're actively working on Riju. + +## Finish AWS configuration + +Go back to the AWS console and take care of a few loose ends: + +* If you want, register a [custom public registry alias for + ECR](https://us-west-1.console.aws.amazon.com/ecr/registries?region=us-west-1). + This will make your public registry URL easier to remember. +* In the "View push commands" modal dialogs, take note of the + repository URLs for your public and private Riju ECR repositories. +* If you want alerts, [create an SNS + subscription](https://us-west-1.console.aws.amazon.com/sns/v3/home?region=us-west-1#/subscriptions) + from the Riju SNS topic to the PagerDuty integration URL. + +## Create local configuration (part 3 of 3) + +Add to `.env` the following contents: ``` -AMI_NAME=riju-1609531301 +# Build +DOCKER_REPO=800516322591.dkr.ecr.us-west-1.amazonaws.com/riju +PUBLIC_DOCKER_REPO=public.ecr.aws/yourname/riju ``` -Next, we will create the rest of the infrastructure. +### DOCKER\_REPO + +This is the URL for your private ECR repository. + +### PUBLIC\_DOCKER\_REPO + +This is the URL for your public ECR repository. + +## Configure DNS + +Obtain the DNS record for Riju's ALB from `terraform output` and +install it as a proxied CNAME record in CloudFlare DNS for your apex +domain. After DNS propagates, you should now be able to receive a 502 +from Riju with no body content. + +## Set up dev server + +The dev server is provisioned with a fresh Ubuntu AMI. You probably +want to clone your repository up there, enable SSH agent forwarding, +etc. Doing a full build on your laptop is feasible, but unless you +have symmetric gigabit ethernet you're not going to get all the build +artifacts uploaded in less than a week. + +## Build and deploy + +Invoke Depgraph: ``` -$ cd ../tf -$ terraform init -backend-config="bucket=riju-yourname-tf" -$ terraform apply +$ dep deploy:live --publish ``` -This will print out the IP address of your newly provisioned server, -as well as permission-limited AWS credentials for use in CI. Add a -line to `.env` with the IP address: - -``` -DOMAIN=54.183.183.91 -``` - -## Bootstrap server - -Your newly provisioned server also isn't running anything yet. -You'll want to bootstrap it with the official image to make sure -everything is in working order: - -``` -$ ./tools/deploy.bash raxod502/riju:app -``` - -This may take a while. After it's finished, however, you should be -able to navigate to the IP address of your server in a browser and see -Riju up and running. - -## Bootstrap S3 - -You probably don't want to build all of Riju's languages from scratch -when deploying a modified version, since that would take a long time. -You can avoid it by copying the latest built languages from Riju's -production S3 bucket. This will be fastest if you SSH into your -production server, which lives in AWS: - -``` -$ ssh admin@your-ip-address -$ export AWS_ACCESS_KEY_ID=... -$ export AWS_SECRET_ACCESS_KEY=... -$ aws s3 cp --recursive --source-region us-west-1 s3://riju-debs s3://riju-yourname-debs -``` +After innumerable hours of build time (and probably some debugging to +fix languages that have broken since the last full build), Riju +should(tm) be live on your domain. You can connect to the live server +using EC2 Instance Connect by retrieving its instance ID from the AWS +console and running `mssh admin@i-theinstanceid`. Then you can check +(using the previously configured admin password) `sudo journalctl -efu +riju` to see the supervisor logs. ## Set up CI -Go to [CircleCI](https://app.circleci.com/dashboard) and enable builds -for your fork of Riju. In the project settings, configure the -following environment variables: +In your GitHub repo settings, create the secrets `AWS_ACCESS_KEY_ID` +and `AWS_SECRET_ACCESS_KEY` with the values from `terraform output +-json`. GitHub Actions should be good to go! However, I would +recommend doing builds from the EC2 dev server when you need to +rebuild a lot of artifacts. -* `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`: the AWS access - credentials generated by Terraform -* `DEPLOY_SSH_PRIVATE_KEY`: base64-encoded SSH private key, e.g. from - `base64 -w0 ~/.ssh/id_rsa_riju_deploy` -* `DOCKER_REPO`: same as in `.env` -* `DOCKER_USERNAME` and `DOCKER_PASSWORD`: your credentials for Docker - Hub -* `DOMAIN`: same as in `.env` -* `S3_BUCKET`: same as in `.env` - -You should now be able to trigger a build and see the production -instance updated automatically with a newly built image. - -## Enable TLS - -By default Riju will serve HTTP traffic if a TLS certificate is not -available. You can fix this. First, you'll need a domain name (or -subdomain on an existing domain). If you don't have one, you can buy -one at e.g. [Namecheap](https://www.namecheap.com/). - -In your domain registrar's configuration interface, go to the -manual/advanced DNS settings and create an A record for your domain -pointing at your server's IP address. (To point the top-level domain -at Riju, set the host to `@`; to point a subdomain -`foo.yourdomain.io`, set the host to `foo`.) - -Now SSH into the production server and run: - -``` -$ sudo systemctl stop riju -$ sudo certbot certonly --standalone -$ riju-install-certbot-hooks -$ sudo systemctl start riju -``` - -Alternatively, if you have an existing Certbot certificate you'd like -to transfer to the new server, you can copy over the entire -`/etc/letsencrypt` direction exactly as it stands, and run -`riju-install-certbot-hooks`. +You'll also want to go to `.github/workflows/main.yml` and update the +environment variables `AWS_REGION`, `DOCKER_REPO`, +`PUBLIC_DOCKER_REPO`, and `S3_BUCKET` as appropriate for your own +deployment (see the `.env` file you created earlier). diff --git a/doc/tutorial.md b/doc/tutorial.md index 9f69c5f..f880e92 100644 --- a/doc/tutorial.md +++ b/doc/tutorial.md @@ -48,6 +48,15 @@ keybindings are: use `control-b` twice to do a command on the inner one instead of the outer one +## Fetch base Ubuntu image + +Make sure you're using the same version of Ubuntu as the mainline +Riju: + +``` +$ make sync-ubuntu +``` + ## Start Riju server Use `dep`, the Riju build tool, to compile the Docker image that the @@ -105,7 +114,7 @@ language: * [Install your language](tutorial/install.md) * [Provide run commands](tutorial/run.md) -* [Configure tests](tutorial/tests.md) -* [Provide metadata](tutorial/metadata.md) * [Add code formatter](tutorial/formatter.md) * [Add language server](tutorial/lsp.md) +* [Configure tests](tutorial/tests.md) +* [Provide metadata](tutorial/metadata.md) diff --git a/doc/tutorial/run.md b/doc/tutorial/run.md index 2c5c42d..920af1f 100644 --- a/doc/tutorial/run.md +++ b/doc/tutorial/run.md @@ -166,3 +166,26 @@ template: | run: | zoem -I main.azm; zoem ``` + +## Required input + +Sometimes languages really don't want to provide a way to run code +noninteractively. In this case, you can include instructions about how +the user can manually run the code: + +``` +repl: | + hhvm -a +input: | + print 123 * 234 + +main: "main.hack" +template: | + <<__EntryPoint>> + function main(): void { + echo "Hello, world!\n"; + } + +run: | + echo "Type 'r' at the debugger prompt to run the code" && hhvm -a main.hack +``` diff --git a/doc/tutorial/tests.md b/doc/tutorial/tests.md index 1333ed7..f3dacde 100644 --- a/doc/tutorial/tests.md +++ b/doc/tutorial/tests.md @@ -1 +1,167 @@ -TODO +# Tutorial: Configure tests + +Riju has far too many languages to rely on manual testing to ensure +that features are not broken by upstream updates. As such, we have a +system to automatically generate test cases which are validated +whenever language configuration changes. + +Here are the tests that can be configured for each language: + +* `run` (mandatory for all languages): Verify that the `template` code + prints `Hello, world!` when run using the `run` command. +* `repl` (mandatory for all languages with `repl`): Verify that + submitting `123 * 234` to the `repl` command causes `28782` to be + printed. +* `runrepl` (mandatory for all languages with `repl`): Same as `repl`, + but using the `run` command instead of the `repl` command (i.e., + test that the `run` command starts a REPL after executing `main`). +* `scope` (optional, only for languages with variable-scope-preserving + `repl`): Verify that if a variable assignment is appended to `main`, + then that variable can be evaluated from the REPL using the `run` + command. +* `format` (mandatory for all languages with `format`): Verify that a + misformatted version of the `template` code is correctly formatted + back to its canonical form when executing `format.run`. +* `lsp` (mandatory for all languages with `lsp`): Verify that the + language server produces a given autocompletion in a given context. +* `ensure` (optional): Verify that a specific shell command executes + successfully. This is currently unused. + +## Test configuration + +See [`jsonschema.yaml`](../../lib/jsonschema.yaml) for full +documentation. + +* `run` + * If your language can't be made to print exactly `Hello, world!`, + specify the actual output using the `hello` key. + * In extraordinary circumstances, a language may be unable to + produce deterministic output (e.g. Entropy). In such cases, + `hello` can also be a JavaScript-compatible regular + expression, and you must specify `helloMaxLength` which is + the maximum possible length of the `Hello, world` output + matched by the regex. + * Some languages require user input at the REPL to run the code, + despite our best efforts to the contrary. In this case, you can + specify the required input in the `helloInput` key. (See + "Specifying user input" below.) + * If a language *doesn't* have `repl`, then the `run` command is + expected to terminate after executing user code. By default the + expected exit status is 0, and the `run` test will fail + otherwise. If for some reason your language exits with a nonzero + status even in the absence of an error, then you can specify the + expected status in the `helloStatus` key. +* `repl` + * We try to compute `123 * 234` in most languages' REPLs. + Naturally, the syntax may vary depending on the language, so you + can specify an alternate input using the `input` key. (See + "Specifying user input" below.) + * The result of `123 * 234` is generally `28782`. In the case that + we get the output in some other format, you can specify the + expected output using the `output` key. +* `runrepl` + * In the case that `input` needs to be different for `runrepl` + than for `repl`, you can override it specifically for `runrepl` + using the `runReplInput` key. + * In the case that `output` needs to be different for `runrepl` + than for `repl`, you can override it specifically for `runrepl` + using the `runReplOutput` key. +* `scope` + * *Required:* In `scope.code`, specify the code that is needed to + assign `123 * 234` to a local variable named `x`, or as close to + that as the language can manage. For example, in Python, this + would be `x = 123 * 234`. + * By default, `scope.code` is appended at the end of `template`. + However, if it needs to go in the middle, you can specify + `scope.after`, which should match an entire line of `template`. + Then `scope.code` will be placed on the next list after + `scope.after`. + * By default, it's expected that typing `x` into the REPL will + produce `28782`. You can override the input to something else by + specifying `scope.input`. (See "Specifying user input" below.) + * If the expected output is something other than `28782`, you can + override it using `scope.output`. +* `format` + * *Required:* In `format.input`, specify the input code. This + should be distinct from `template`, but should turn into + `template` when the `format` command is run on it. + * In the case that you can't come up with input that formats to + `template`, you can specify `format.output` as the expected + result of formatting `format.input`. +* `lsp` + * *Required:* In `lsp.code`, specify input (not necessarily an + entire line) that we should pretend the user has typed. We + expect an autocompletion to be presented with the cursor at the + end of this input. + * *Required:* In `lsp.item`, specify the text of an autocompletion + that we expect the language server to generate. This should not + match any text that actually appears in `template` or + `lsp.code`. + * In `lsp.after`, you can specify a string that will match exactly + one place in `template`. This is where the cursor will be + positioned before `lsp.code` is inserted. If `lsp.after` is not + specified, then `lsp.code` will be inserted at the end of + `template` on a new line. +* `ensure` + * If you want to use an `ensure` test, just supply the shell + command using the `ensure` key. + +## Specifying user input + +We have a couple different formats for common types of user input. +This will type `eval` and send a newline: + +```yaml +input: | + eval +``` + +This will type `eval`, send a newline, then type `go` and send another +newline: + +```yaml +input: | + eval + go +``` + +This will type `foo`, send a newline, wait 1 second, then type `bar` +and send another newline: + +```yaml +input: | + foo + DELAY: 1 + bar +``` + +Why is this useful? Unfortunately, many languages have race conditions +and will fail to notice input if you send it before they have finished +starting up. + +Finally, this will type `foo` and then send a newline followed by an +EOF: + +```yaml +input: | + foo + EOF +``` + +## Broken tests + +We try very hard to get tests working for every newly added language. +However, sometimes there's something truly puzzling going on that's +not worth blocking a new language being added. For that reason it's +possible to mark a test as temporarily skipped, e.g.: + +```yaml +skip: + - repl + - runrepl + - scope + - lsp +``` + +This is unfortunately currently the case for many of the LSP tests due +to the fragility of most language servers. diff --git a/docker/admin/install.bash b/docker/admin/install.bash index 5bf500a..e4495d1 100755 --- a/docker/admin/install.bash +++ b/docker/admin/install.bash @@ -48,6 +48,7 @@ moreutils nodejs packer psmisc +python3-pip pwgen skopeo ssh @@ -65,6 +66,8 @@ yarn apt-get update apt-get install -y $(sed 's/#.*//' <<< "${packages}") +pip3 install ec2instanceconnectcli + wget -nv https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O awscli.zip unzip -q awscli.zip ./aws/install diff --git a/frontend/pages/app.ejs b/frontend/pages/app.ejs index eb5f385..cfa8e3c 100644 --- a/frontend/pages/app.ejs +++ b/frontend/pages/app.ejs @@ -23,8 +23,8 @@ window.rijuConfig = <%- JSON.stringify(config) %>; - <% if (analyticsEnabled) { %> - + <% if (fathomSiteId) { %> + <% } %> diff --git a/frontend/pages/index.ejs b/frontend/pages/index.ejs index 252f498..77c1830 100644 --- a/frontend/pages/index.ejs +++ b/frontend/pages/index.ejs @@ -30,8 +30,8 @@ <% } else { %> Riju is loading language configuration... <% } %> - <% if (analyticsEnabled) { %> - + <% if (fathomSiteId) { %> + <% } %> diff --git a/langs/abc.yaml b/langs/abc.yaml index bf8cdb5..6f80b2b 100644 --- a/langs/abc.yaml +++ b/langs/abc.yaml @@ -20,7 +20,7 @@ install: repl: | abc input: | - DELAY: 1 + DELAY: 2 WRITE 123 * 234 main: "main.abc" diff --git a/langs/purescript.yaml b/langs/purescript.yaml index bca1f49..b9db86c 100644 --- a/langs/purescript.yaml +++ b/langs/purescript.yaml @@ -17,29 +17,38 @@ install: - purescript - spago manual: | - install -d "${pkg}/opt/purescript" + install -d "${pkg}/opt/purescript/skel-home" + install -d "${pkg}/opt/purescript/skel-src" - mkdir skel - pushd skel spago init -C rm -rf .gitignore test sed -i 's#, "test/\*\*/\*\.purs"##' spago.dhall - cat <<"EOF" > src/Main.spago + + cat <<"EOF" > src/Main.purs + module Main where + import Prelude import Effect (Effect) + import Effect.Console (log) main :: Effect Unit - main = pure unit + main = log "Hello, world!" EOF + spago build spago repl < /dev/null + rm -rf src - popd - cp -R skel "${pkg}/opt/purescript/" + + shopt -s dotglob + cp -R --preserve=timestamps * "${pkg}/opt/purescript/skel-src/" + cp -R --preserve=timestamps "${HOME}/.cache" "${pkg}/opt/purescript/skel-home/" setup: | - shopt -s dotglob; cp -R /opt/purescript/skel/* "$PWD/" + shopt -s dotglob + cp -R --preserve=timestamps /opt/purescript/skel-home/* "${HOME}/" + cp -R --preserve=timestamps /opt/purescript/skel-src/* "${PWD}/" repl: | spago repl diff --git a/lib/jsonschema.yaml b/lib/jsonschema.yaml index a8bf7ee..84f32c8 100644 --- a/lib/jsonschema.yaml +++ b/lib/jsonschema.yaml @@ -902,7 +902,7 @@ properties: format: type: object additionalProperties: false - required: [run] + required: [run, input] properties: run: type: string diff --git a/packer/provision.bash b/packer/provision.bash index f1d153e..d047734 100644 --- a/packer/provision.bash +++ b/packer/provision.bash @@ -51,6 +51,7 @@ sudo sed -Ei 's/^#?PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config sudo sed -Ei 's/^#?PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config sudo sed -Ei 's/^#?PermitEmptyPasswords .*/PermitEmptyPasswords no/' /etc/ssh/sshd_config sudo sed -Ei "s/\\\$AWS_REGION/${AWS_REGION}/" /etc/systemd/system/riju.service +sudo sed -Ei "s/\\\$FATHOM_SITE_ID/${FATHOM_SITE_ID:-}/" /etc/systemd/system/riju.service sudo sed -Ei "s/\\\$S3_BUCKET/${S3_BUCKET}/" /etc/systemd/system/riju.service sudo sed -Ei "s/\\\$SUPERVISOR_ACCESS_TOKEN/${SUPERVISOR_ACCESS_TOKEN}/" /etc/systemd/system/riju.service diff --git a/packer/riju.service b/packer/riju.service index c272d02..cf880b3 100644 --- a/packer/riju.service +++ b/packer/riju.service @@ -11,6 +11,7 @@ ExecStart=riju-supervisor Restart=always RestartSec=5 Environment=AWS_REGION=$AWS_REGION +Environment=FATHOM_SITE_ID=$FATHOM_SITE_ID Environment=S3_BUCKET=$S3_BUCKET Environment=SUPERVISOR_ACCESS_TOKEN=$SUPERVISOR_ACCESS_TOKEN diff --git a/supervisor/src/main.go b/supervisor/src/main.go index d580e88..096a030 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -347,11 +347,13 @@ func (sv *supervisor) reload() error { "-v", "/var/run/riju:/var/run/riju", "-v", "/var/run/docker.sock:/var/run/docker.sock", "-p", fmt.Sprintf("127.0.0.1:%d:6119", port), + "-e", "FATHOM_SITE_ID", "-e", "RIJU_DEPLOY_CONFIG", - "-e", "ANALYTICS=1", "--label", fmt.Sprintf("riju.deploy-config-hash=%s", deployCfgHash), "--name", name, - "--restart=unless-stopped", + "--restart", "unless-stopped", + "--oom-kill-disable", + "--cpu-shares", "2048", fmt.Sprintf("riju:%s", deployCfg.AppImageTag), ) dockerRun.Stdout = os.Stdout diff --git a/system/src/riju-system-privileged.c b/system/src/riju-system-privileged.c index de6d775..7f089bb 100644 --- a/system/src/riju-system-privileged.c +++ b/system/src/riju-system-privileged.c @@ -107,9 +107,9 @@ void session(char *uuid, char *lang, char *imageHash) "--user", "root", "--hostname", lang, "--name", container, - "--cpus", "0.25", - "--memory", "100m", - "--memory-swap", "900m", + "--cpus", "1", + "--memory", "1g", + "--memory-swap", "3g", image, "bash", "-c", "cat /var/run/riju/sentinel/fifo | ( sleep 10; while read -t2; do :; done; pkill -g0 )", NULL, diff --git a/tf/main.tf b/tf/main.tf index afa2bd9..f04bf9a 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -1,7 +1,6 @@ terraform { backend "s3" { key = "state" - region = "us-west-1" } required_providers { aws = { @@ -30,7 +29,6 @@ locals { } provider "aws" { - region = "us-west-1" default_tags { tags = local.tags } diff --git a/tools/ci-run.bash b/tools/ci-run.bash index 8658ebb..f34dd51 100755 --- a/tools/ci-run.bash +++ b/tools/ci-run.bash @@ -2,5 +2,5 @@ set -euo pipefail -make ecr system +make ecr make env CMD="dep deploy:live --publish --yes" Z=xz CI=1 TEST_PATIENCE=4 TEST_CONCURRENCY=1 diff --git a/tools/depgraph.js b/tools/depgraph.js index 6fe740b..abf2fa2 100644 --- a/tools/depgraph.js +++ b/tools/depgraph.js @@ -276,7 +276,7 @@ async function getDeployLiveArtifact(langs) { dependencies: ["deploy:ready"], publishTarget: true, publishToRegistry: async () => { - await runCommand(`make deploy`); + await runCommand(`make deploy-latest`); }, }; } @@ -679,7 +679,7 @@ async function main() { if (program.args.length === 0) { program.help({ error: true }); } - await runCommand("make all-scripts"); + await runCommand("make all-scripts system"); await executeDepGraph({ depgraph, manual,