Export metrics to Grafana Cloud

packer/node-exporter.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Prometheus node exporter
+StartLimitBurst=5
+StartLimitIntervalSec=300
+
+[Service]
+Type=exec
+ExecStart=node_exporter
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

packer/prometheus.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Prometheus
+StartLimitBurst=5
+StartLimitIntervalSec=300
+
+[Service]
+Type=exec
+ExecStart=prometheus --config.file /etc/prometheus/config.yaml
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

packer/prometheus.yaml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: server
+    static_configs:
+      - targets: ["localhost:6121"]
+  - job_name: node
+    static_configs:
+      - targets: ["localhost:9100"]
+  - job_name: prometheus
+    static_configs:
+      - targets: ["localhost:9090"]
+
+remote_write:
+  - url: "https://prometheus-blocks-prod-us-central1.grafana.net/api/prom/push"
+    basic_auth:
+      username: "$GRAFANA_PROMETHEUS_USERNAME"
+      password: "$GRAFANA_API_KEY"

packer/promtail.yaml

@@ -7,7 +7,7 @@ positions:
   filename: /tmp/positions.yaml
 
 client:
-  url: https://72217:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push
+  url: https://$GRAFANA_LOKI_USERNAME:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push
 
 scrape_configs:
   - job_name: kernel

packer/provision-web.bash

@@ -77,16 +77,38 @@ if [[ -n "${GRAFANA_API_KEY:-}" ]]; then
     unzip promtail-linux-amd64.zip
     sudo cp promtail-linux-amd64 /usr/local/bin/promtail
 
-    sudo chown root:root /tmp/promtail.service /tmp/promtail.yaml
+    ver="$(latest_release prometheus/node_exporter | sed 's/^v//')"
 
-    sudo mkdir /etc/promtail
+    wget -nv "https://github.com/prometheus/node_exporter/releases/download/v${ver}/node_exporter-${ver}.linux-amd64.tar.gz" -O node_exporter.tar.gz
+    tar -xf node_exporter.tar.gz --strip-components=1
+    sudo cp node_exporter /usr/local/bin/
+
+    ver="$(latest_release prometheus/prometheus | sed 's/^v//')"
+
+    wget -nv "https://github.com/prometheus/prometheus/releases/download/v${ver}/prometheus-${ver}.linux-amd64.tar.gz" -O prometheus.tar.gz
+    tar -xf prometheus.tar.gz --strip-components=1
+    sudo cp prometheus /usr/local/bin/
+
+    sudo chown root:root                                                \
+         /tmp/node-exporter.service /tmp/prometheus.service             \
+         /tmp/prometheus.yaml /tmp/promtail.service /tmp/promtail.yaml
+
+    sudo mkdir /etc/prometheus /etc/promtail
+    sudo mv /tmp/prometheus.yaml /etc/prometheus/config.yaml
     sudo mv /tmp/promtail.yaml /etc/promtail/config.yaml
-    sudo mv /tmp/promtail.service /etc/systemd/system/
+    sudo mv /tmp/prometheus.service /tmp/promtail.service /tmp/node-exporter.service \
-    sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" /etc/promtail/config.yaml
+         /etc/systemd/system/
 
-    sudo systemctl enable promtail
+    sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" \
+         /etc/prometheus/config.yaml /etc/promtail/config.yaml
+    sudo sed -Ei "s/\\\$GRAFANA_LOKI_USERNAME/${GRAFANA_LOKI_USERNAME}/" \
+         /etc/promtail/config.yaml
+    sudo sed -Ei "s/\\\$GRAFANA_PROMETHEUS_USERNAME/${GRAFANA_PROMETHEUS_USERNAME}/" \
+         /etc/prometheus/config.yaml
+
+    sudo systemctl enable node-exporter prometheus promtail
 else
-    sudo rm /tmp/promtail.yaml /tmp/promtail.service
+    sudo rm /tmp/node-exporter.service /tmp/promtail.yaml /tmp/promtail.service
 fi
 
 sudo userdel ubuntu -f

packer/web.pkr.hcl

@@ -13,6 +13,16 @@ variable "analytics_tag" {
   default = "${env("ANALYTICS_TAG")}"
 }
 
+variable "grafana_loki_username" {
+  type    = string
+  default = "${env("GRAFANA_LOKI_USERNAME")}"
+}
+
+variable "grafana_prometheus_username" {
+  type    = string
+  default = "${env("GRAFANA_PROMETHEUS_USERNAME")}"
+}
+
 variable "grafana_api_key" {
   type    = string
   default = "${env("GRAFANA_API_KEY")}"
@@ -82,6 +92,21 @@ build {
     source = "docker.json"
   }
 
+  provisioner "file" {
+    destination = "/tmp/node-exporter.service"
+    source = "node-exporter.service"
+  }
+
+  provisioner "file" {
+    destination = "/tmp/prometheus.service"
+    source = "prometheus.service"
+  }
+
+  provisioner "file" {
+    destination = "/tmp/prometheus.yaml"
+    source = "prometheus.yaml"
+  }
+
   provisioner "file" {
     destination = "/tmp/promtail.service"
     source = "promtail.service"
@@ -117,6 +142,8 @@ build {
       "ADMIN_PASSWORD=${var.admin_password}",
       "AWS_REGION=${var.aws_region}",
       "ANALYTICS_TAG=${var.analytics_tag}",
+      "GRAFANA_LOKI_USERNAME=${var.grafana_loki_username}",
+      "GRAFANA_PROMETHEUS_USERNAME=${var.grafana_prometheus_username}",
       "GRAFANA_API_KEY=${var.grafana_api_key}",
       "S3_BUCKET=${var.s3_bucket}",
       "SENTRY_DSN=${var.sentry_dsn}",

supervisor/src/main.go

@@ -355,6 +355,7 @@ func (sv *supervisor) reload() error {
 		"-v", "/var/cache/riju:/var/cache/riju",
 		"-v", "/var/run/docker.sock:/var/run/docker.sock",
 		"-p", fmt.Sprintf("127.0.0.1:%d:6119", port),
+		"-p", "127.0.0.1:6121:6121",
 		"-e", "ANALYTICS_TAG",
 		"-e", "RIJU_DEPLOY_CONFIG",
 		"-e", "SENTRY_DSN",