mirror of
https://github.com/Blockstream/satellite-api.git
synced 2025-02-21 13:24:03 +01:00
add prom module
This commit is contained in:
nitram
parent
8912748f24
commit
8badc6a9ea
5 changed files with 412 additions and 0 deletions
412
terraform/modules/prometheus/cloud-init/prometheus.yml
Normal file
412
terraform/modules/prometheus/cloud-init/prometheus.yml
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
bootcmd:
|
||||
- blkid /dev/disk/by-id/google-data || mkfs.ext4 -L data -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-data
|
||||
|
||||
mounts:
|
||||
- [ /dev/disk/by-label/data, /mnt/disks/data, auto, defaults ]
|
||||
|
||||
users:
|
||||
- name: exec
|
||||
uid: 2000
|
||||
|
||||
write_files:
|
||||
- path: /home/exec/prometheus/alertmanager.yml
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
route:
|
||||
group_by: [cluster, alertname]
|
||||
# If an alert isn't caught by a route, send it to the pager.
|
||||
receiver: noc-pager
|
||||
routes:
|
||||
- match:
|
||||
severity: warning
|
||||
receiver: noc-email
|
||||
- match:
|
||||
severity: page
|
||||
receiver: noc-pager
|
||||
|
||||
receivers:
|
||||
- name: noc-email
|
||||
email_configs:
|
||||
- to: noc@blockstream.io
|
||||
from: noreply@blockstream.io
|
||||
smarthost: smtp.gmail.com:587
|
||||
auth_username: ${email_username}
|
||||
auth_password: ${email_password}
|
||||
- name: noc-pager
|
||||
opsgenie_configs:
|
||||
- api_key: ${opsgenie_api_key}
|
||||
teams: SecOps
|
||||
tags: lightning-store
|
||||
|
||||
- path: /home/exec/prometheus/rules/alerts.yml
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
groups:
|
||||
- name: node
|
||||
rules:
|
||||
#- alert: NoHostsInNetwork
|
||||
# expr: sum by (name) (up{name=~".+"}) == 0
|
||||
# for: 1m
|
||||
# labels:
|
||||
# severity: page
|
||||
# annotations:
|
||||
# summary: No hosts in network {{ $labels.name }}, production traffic impacted!
|
||||
# description: There are currently no hosts up in the network {{ $labels.name }}, verify the instance groups. https://wiki.blockstream.io/OpsPlaybooks/Esplora-Runbooks#NoHostsInNetwork
|
||||
|
||||
- path: /home/exec/prometheus/prometheus.yml
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
global:
|
||||
evaluation_interval: 15s
|
||||
scrape_interval: 15s
|
||||
external_labels:
|
||||
project: blockstream-store
|
||||
|
||||
rule_files:
|
||||
- /config/rules/alerts.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- scheme: http
|
||||
static_configs:
|
||||
- targets:
|
||||
- "127.0.0.1:9093"
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
- '__meta_gce_label_network'
|
||||
target_label: 'network'
|
||||
- source_labels:
|
||||
- '__meta_gce_label_name'
|
||||
target_label: 'name'
|
||||
- source_labels:
|
||||
- '__meta_gce_instance_name'
|
||||
target_label: 'instance_name'
|
||||
gce_sd_configs:
|
||||
- project: blockstream-store
|
||||
zone: asia-northeast1-a
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: asia-northeast1-b
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: asia-northeast1-c
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: europe-west4-a
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: europe-west4-b
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: europe-west4-c
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-central1-a
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-central1-b
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-central1-c
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-central1-f
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-east1-d
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-east1-b
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
zone: us-east1-c
|
||||
filter: (labels.type = "prometheus")
|
||||
port: 9100
|
||||
- job_name: wordpress
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
- '__meta_gce_label_network'
|
||||
target_label: 'network'
|
||||
- source_labels:
|
||||
- '__meta_gce_label_name'
|
||||
target_label: 'name'
|
||||
- source_labels:
|
||||
- '__meta_gce_instance_name'
|
||||
target_label: 'instance_name'
|
||||
gce_sd_configs:
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: asia-northeast1-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: asia-northeast1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: asia-northeast1-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: europe-west4-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: europe-west4-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: europe-west4-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-central1-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-central1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-central1-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-central1-f
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-east1-d
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-east1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "wordpress")
|
||||
zone: us-east1-c
|
||||
port: 9100
|
||||
- job_name: satellite-api
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
- '__meta_gce_label_network'
|
||||
target_label: 'network'
|
||||
- source_labels:
|
||||
- '__meta_gce_label_name'
|
||||
target_label: 'name'
|
||||
- source_labels:
|
||||
- '__meta_gce_instance_name'
|
||||
target_label: 'instance_name'
|
||||
gce_sd_configs:
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: asia-northeast1-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: asia-northeast1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: asia-northeast1-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: europe-west4-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: europe-west4-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: europe-west4-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-central1-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-central1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-central1-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-central1-f
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-east1-d
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-east1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-east1-c
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-west1-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-west1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "lightning-app")
|
||||
zone: us-west1-c
|
||||
port: 9100
|
||||
- job_name: btcpayserver
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
- '__meta_gce_label_network'
|
||||
target_label: 'network'
|
||||
- source_labels:
|
||||
- '__meta_gce_label_name'
|
||||
target_label: 'name'
|
||||
- source_labels:
|
||||
- '__meta_gce_instance_name'
|
||||
target_label: 'instance_name'
|
||||
gce_sd_configs:
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "btcpayserver")
|
||||
zone: us-west1-a
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "btcpayserver")
|
||||
zone: us-west1-b
|
||||
port: 9100
|
||||
- project: blockstream-store
|
||||
filter: (labels.type = "btcpayserver")
|
||||
zone: us-west1-c
|
||||
port: 9100
|
||||
|
||||
- path: /etc/systemd/system/prometheus.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=prometheus-server instance
|
||||
Wants=gcr-online.target docker.service
|
||||
After=gcr-online.service docker.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=1
|
||||
Environment=HOME=/home/exec
|
||||
ExecStartPre=/usr/bin/docker-credential-gcr configure-docker
|
||||
ExecStartPre=/usr/bin/docker pull ${docker_tag}
|
||||
ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 9090 -j ACCEPT
|
||||
ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 80 -j ACCEPT
|
||||
ExecStartPre=/sbin/iptables -A PREROUTING -t nat -p tcp --dport 80 -j REDIRECT --to-port 9090
|
||||
ExecStart=/usr/bin/docker run \
|
||||
--network=host \
|
||||
-v /mnt/disks/data:/data:rw \
|
||||
-v /home/exec/prometheus:/config:ro \
|
||||
--read-only \
|
||||
--name prometheus \
|
||||
"${docker_tag}" --config.file=/config/prometheus.yml --web.enable-lifecycle --web.enable-admin-api --storage.tsdb.path=/data/metrics --storage.tsdb.retention=${retention}
|
||||
ExecStop=/usr/bin/docker stop prometheus
|
||||
ExecStopPost=/usr/bin/docker rm prometheus
|
||||
ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 9090 -j ACCEPT
|
||||
ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 80 -j ACCEPT
|
||||
ExecStopPost=/sbin/iptables -D PREROUTING -t nat -p tcp --dport 80 -j REDIRECT --to-port 9090
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- path: /etc/systemd/system/alertmanager.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=alertmanager-server instance
|
||||
Wants=gcr-online.target docker.service
|
||||
After=gcr-online.service docker.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=1
|
||||
Environment=HOME=/home/exec
|
||||
ExecStartPre=/usr/bin/docker-credential-gcr configure-docker
|
||||
ExecStartPre=/usr/bin/docker pull ${docker_tag}
|
||||
ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 9093 -j ACCEPT
|
||||
ExecStart=/usr/bin/docker run \
|
||||
--network=host \
|
||||
-v /mnt/disks/data:/data:rw \
|
||||
-v /home/exec/prometheus:/config:ro \
|
||||
--read-only \
|
||||
--name alertmanager \
|
||||
--entrypoint=/bin/alertmanager \
|
||||
"${docker_tag}" --config.file=/config/alertmanager.yml
|
||||
ExecStop=/usr/bin/docker stop alertmanager
|
||||
ExecStopPost=/usr/bin/docker rm alertmanager
|
||||
ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 9093 -j ACCEPT
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- path: /etc/systemd/system/node-exporter.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=prometheus node-exporter
|
||||
Wants=gcr-online.target docker.service
|
||||
After=gcr-online.service docker.service
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=1
|
||||
Environment=HOME=/home/exec
|
||||
ExecStartPre=/usr/bin/docker-credential-gcr configure-docker
|
||||
ExecStartPre=/usr/bin/docker pull ${docker_tag_node_exporter}
|
||||
ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 9100 -j ACCEPT
|
||||
ExecStart=/usr/bin/docker run \
|
||||
--name=node-exporter \
|
||||
--network=host \
|
||||
--read-only \
|
||||
-v /proc:/host/proc:ro \
|
||||
-v /sys:/host/sys:ro \
|
||||
-v /:/rootfs:ro \
|
||||
-v metrics:/metrics:ro \
|
||||
"${docker_tag_node_exporter}" --path.procfs /host/proc --path.sysfs /host/sys --collector.textfile.directory /metrics --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc($|/))"
|
||||
ExecStop=/usr/bin/docker stop node-exporter
|
||||
ExecStopPost=/usr/bin/docker rm node-exporter
|
||||
ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 9100 -j ACCEPT
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
runcmd:
|
||||
- /bin/mkdir -p /mnt/disks/data/metrics
|
||||
- /bin/chown nobody:nobody /mnt/disks/data/metrics
|
||||
- systemctl daemon-reload
|
||||
- systemctl start prometheus.service
|
||||
- systemctl enable prometheus.service
|
||||
- systemctl start alertmanager.service
|
||||
- systemctl enable alertmanager.service
|
||||
- systemctl start node-exporter.service
|
||||
- systemctl enable node-exporter.service
|
||||
0
terraform/modules/prometheus/data.tf
Normal file
0
terraform/modules/prometheus/data.tf
Normal file
0
terraform/modules/prometheus/iam.tf
Normal file
0
terraform/modules/prometheus/iam.tf
Normal file
0
terraform/modules/prometheus/main.tf
Normal file
0
terraform/modules/prometheus/main.tf
Normal file
0
terraform/modules/prometheus/variables.tf
Normal file
0
terraform/modules/prometheus/variables.tf
Normal file
Loading…
Add table
Reference in a new issue