diff --git a/packer/node-exporter.service b/packer/node-exporter.service new file mode 100644 index 0000000..5a42d9f --- /dev/null +++ b/packer/node-exporter.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus node exporter +StartLimitBurst=5 +StartLimitIntervalSec=300 + +[Service] +Type=exec +ExecStart=node_exporter +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packer/prometheus.service b/packer/prometheus.service new file mode 100644 index 0000000..043c45b --- /dev/null +++ b/packer/prometheus.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prometheus +StartLimitBurst=5 +StartLimitIntervalSec=300 + +[Service] +Type=exec +ExecStart=prometheus --config.file /etc/prometheus/config.yaml +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/packer/prometheus.yaml b/packer/prometheus.yaml new file mode 100644 index 0000000..fbc2550 --- /dev/null +++ b/packer/prometheus.yaml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: server + static_configs: + - targets: ["localhost:6121"] + - job_name: node + static_configs: + - targets: ["localhost:9100"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + +remote_write: + - url: "https://prometheus-blocks-prod-us-central1.grafana.net/api/prom/push" + basic_auth: + username: "$GRAFANA_PROMETHEUS_USERNAME" + password: "$GRAFANA_API_KEY" diff --git a/packer/promtail.yaml b/packer/promtail.yaml index f2e1c5d..9edf4e3 100644 --- a/packer/promtail.yaml +++ b/packer/promtail.yaml @@ -7,7 +7,7 @@ positions: filename: /tmp/positions.yaml client: - url: https://72217:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push + url: https://$GRAFANA_LOKI_USERNAME:$GRAFANA_API_KEY@logs-prod-us-central1.grafana.net/api/prom/push scrape_configs: - job_name: kernel diff --git a/packer/provision-web.bash b/packer/provision-web.bash index dac1f74..a23ddfb 100644 --- a/packer/provision-web.bash +++ b/packer/provision-web.bash @@ -77,16 +77,38 @@ if [[ -n "${GRAFANA_API_KEY:-}" ]]; then unzip promtail-linux-amd64.zip sudo cp promtail-linux-amd64 /usr/local/bin/promtail - sudo chown root:root /tmp/promtail.service /tmp/promtail.yaml + ver="$(latest_release prometheus/node_exporter | sed 's/^v//')" - sudo mkdir /etc/promtail + wget -nv "https://github.com/prometheus/node_exporter/releases/download/v${ver}/node_exporter-${ver}.linux-amd64.tar.gz" -O node_exporter.tar.gz + tar -xf node_exporter.tar.gz --strip-components=1 + sudo cp node_exporter /usr/local/bin/ + + ver="$(latest_release prometheus/prometheus | sed 's/^v//')" + + wget -nv "https://github.com/prometheus/prometheus/releases/download/v${ver}/prometheus-${ver}.linux-amd64.tar.gz" -O prometheus.tar.gz + tar -xf prometheus.tar.gz --strip-components=1 + sudo cp prometheus /usr/local/bin/ + + sudo chown root:root \ + /tmp/node-exporter.service /tmp/prometheus.service \ + /tmp/prometheus.yaml /tmp/promtail.service /tmp/promtail.yaml + + sudo mkdir /etc/prometheus /etc/promtail + sudo mv /tmp/prometheus.yaml /etc/prometheus/config.yaml sudo mv /tmp/promtail.yaml /etc/promtail/config.yaml - sudo mv /tmp/promtail.service /etc/systemd/system/ - sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" /etc/promtail/config.yaml + sudo mv /tmp/prometheus.service /tmp/promtail.service /tmp/node-exporter.service \ + /etc/systemd/system/ - sudo systemctl enable promtail + sudo sed -Ei "s/\\\$GRAFANA_API_KEY/${GRAFANA_API_KEY}/" \ + /etc/prometheus/config.yaml /etc/promtail/config.yaml + sudo sed -Ei "s/\\\$GRAFANA_LOKI_USERNAME/${GRAFANA_LOKI_USERNAME}/" \ + /etc/promtail/config.yaml + sudo sed -Ei "s/\\\$GRAFANA_PROMETHEUS_USERNAME/${GRAFANA_PROMETHEUS_USERNAME}/" \ + /etc/prometheus/config.yaml + + sudo systemctl enable node-exporter prometheus promtail else - sudo rm /tmp/promtail.yaml /tmp/promtail.service + sudo rm /tmp/node-exporter.service /tmp/promtail.yaml /tmp/promtail.service fi sudo userdel ubuntu -f diff --git a/packer/web.pkr.hcl b/packer/web.pkr.hcl index c20557a..1268e95 100644 --- a/packer/web.pkr.hcl +++ b/packer/web.pkr.hcl @@ -13,6 +13,16 @@ variable "analytics_tag" { default = "${env("ANALYTICS_TAG")}" } +variable "grafana_loki_username" { + type = string + default = "${env("GRAFANA_LOKI_USERNAME")}" +} + +variable "grafana_prometheus_username" { + type = string + default = "${env("GRAFANA_PROMETHEUS_USERNAME")}" +} + variable "grafana_api_key" { type = string default = "${env("GRAFANA_API_KEY")}" @@ -82,6 +92,21 @@ build { source = "docker.json" } + provisioner "file" { + destination = "/tmp/node-exporter.service" + source = "node-exporter.service" + } + + provisioner "file" { + destination = "/tmp/prometheus.service" + source = "prometheus.service" + } + + provisioner "file" { + destination = "/tmp/prometheus.yaml" + source = "prometheus.yaml" + } + provisioner "file" { destination = "/tmp/promtail.service" source = "promtail.service" @@ -117,6 +142,8 @@ build { "ADMIN_PASSWORD=${var.admin_password}", "AWS_REGION=${var.aws_region}", "ANALYTICS_TAG=${var.analytics_tag}", + "GRAFANA_LOKI_USERNAME=${var.grafana_loki_username}", + "GRAFANA_PROMETHEUS_USERNAME=${var.grafana_prometheus_username}", "GRAFANA_API_KEY=${var.grafana_api_key}", "S3_BUCKET=${var.s3_bucket}", "SENTRY_DSN=${var.sentry_dsn}", diff --git a/supervisor/src/main.go b/supervisor/src/main.go index 2600475..7aaeeb6 100644 --- a/supervisor/src/main.go +++ b/supervisor/src/main.go @@ -355,6 +355,7 @@ func (sv *supervisor) reload() error { "-v", "/var/cache/riju:/var/cache/riju", "-v", "/var/run/docker.sock:/var/run/docker.sock", "-p", fmt.Sprintf("127.0.0.1:%d:6119", port), + "-p", "127.0.0.1:6121:6121", "-e", "ANALYTICS_TAG", "-e", "RIJU_DEPLOY_CONFIG", "-e", "SENTRY_DSN",