Add optional prometheus/grafana monitoring stack (#74)

ml-evs · web-flow · commit e240605b35b8 · 2025-10-23T10:21:55.000+02:00
diff --git a/.vault-pass.sh b/.vault-pass.sh
@@ -2,4 +2,4 @@
 # This script retrieves the password for the current git repository from a Bitwarden Vault.
 # If bw is not available, it should error out
 set -e
-bw get password "$(git remote get-url origin | awk '{split($0, a, "/"); print a[length(a)]}')"
+bw get password "$(git remote get-url origin | awk '{split($0, a, "/"); print a[length(a)]}')" || { read -sp "Password or Bitwarden CLI not found. Enter vault password: " password; echo "$password"; }
diff --git a/README.md b/README.md
@@ -132,6 +132,9 @@ ungrouped:
       borg_encryption_passphrase: <the passphrase for the borg encryption>
       borg_remote_path: <the command to run borg on the repository (e.g., borg1 vs borg2)>
       borg_repository: <the path to the borg repository, either local or remote>
+      prometheus_remote_write_url: <your_prometheus_instance_url, e.g., https://grafana.datalab.industries/prometheus/api/v1/write>
+      prometheus_user: <your_prometheus_username>
+      prometheus_password: <your_prometheus_password>
 ```
 
 where `<hostname>` and the various setting should be configured with your chosen
@@ -259,6 +262,15 @@ version to use the same name using:
 git remote set-url origin <my-git-repo-url>
 ```
 
+You can also simply edit the `.vault-pass.sh` script to return your vault
+password in another way if you prefer.
+
+You may also need to set the script to be executable with
+
+```shell
+chmod u+x .vault-pass.sh
+```
+
 #### Backups
 
 ##### Native backups
@@ -297,6 +309,36 @@ to set up any extra settings (e.g., proxies, host checking), or an `.ssh/known_h
 > for your encrypted Borg backups, feel free to reach out to us as we may have
 > enough of an overhead to be a secondary backup host for you.
 
+#### Server monitoring
+
+One basic option for uptime monitoring is to use a free GitHub Actions based
+service like [Upptime](https://github.com/upptime/upptime).
+For example, this is used for simple services in the central *datalab* organisation at [datalab-org/datalab-org-status](https://github.com/datalab-org/datalab-org-status).
+
+For more advanced monitoring, the Ansible playbooks contain a role tagged as
+`monitoring`, which will install and configure metrics harvesters using
+[Prometheus](https://prometheus.io/) (with [Node Exporter](https://github.com/prometheus/node_exporter) and
+[cAdvisor](https://github.com/google/cadvisor)) to monitor the host system and
+containers.
+
+To make use of this monitoring, you will need your own [Grafana instance](https://grafana.com/oss/grafana) (also running Prometheus as a harvester of the remote metrics) to visualise the metrics.
+
+Alternatively, you can use a hosted Grafana service such as [Grafana Cloud](https://grafana.com/products/cloud/), or request to use our central
+*datalab* Grafana instance by reaching out to us on Slack or over email.
+
+This integration can be enabled by adding the following variables to your inventory:
+
+```yaml
+prometheus_remote_write_url: <your_prometheus_instance_url, e.g., https://grafana.datalab.industries/prometheus/api/v1/write>
+prometheus_user: <your_prometheus_username>
+prometheus_password: <your_prometheus_password>
+```
+and then running the playbook with the `monitoring` tag:
+
+```shell
+make monitoring
+```
+
 ### Cloud provisioning
 
 These instructions will use OpenTofu, an open source fork of Terraform.
diff --git a/ansible/inventory.yml b/ansible/inventory.yml
@@ -4,10 +4,14 @@ ungrouped:
     <hostname>:
       ansible_become_method: sudo
       ansible_user: <remote_username>
+      datalab_prefix: <desired_datalab_prefix (used for monitoring labels)>
       api_url: <desired_datalab_api_url>
       app_url: <desired_datalab_app_url>
       mount_data_disk: <disk device file location, e.g., /dev/sda, /dev/sdb or otherwise>
       data_disk_type: <the fstype of the data disk, defaults to 'xfs'
       borg_encryption_passphrase: <the passphrase for the borg encryption>
       borg_remote_path: <the command to run borg on the repository (e.g., borg1 vs borg2)>
       borg_repository: <the path to the borg repository, either local or remote>
+      prometheus_remote_write_url: <your_prometheus_instance_url, e.g., https://grafana.datalab.industries/prometheus/api/v1/write>
+      prometheus_user: <the username to access the prometheus server>
+      prometheus_password: <the password to access the prometheus server>
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
@@ -29,6 +29,9 @@
     - role: borg
       name: Configure borg(matic) and remote backups
       tags: [borg]
+    - role: monitoring
+      name: Install and configure prometheus monitoring stack
+      tags: [monitoring]
 
   tasks:
     - name: Keep all packages up-to-date
diff --git a/ansible/roles/borg/tasks/main.yml b/ansible/roles/borg/tasks/main.yml
@@ -1,94 +1,97 @@
 ---
-- name: Synchronize borgmatic files to remote
-  ansible.posix.synchronize:
-    src: "{{ role_path }}/files/"
-    dest: "{{ ansible_user_home_dir }}/borgmatic"
-
-- name: Check whether ssh config exists
+- name: Check whether ssh cert for borg exists
   ansible.builtin.stat:
     path: "{{ playbook_dir }}/vaults/borg/.ssh/id_ed25519"
   register: ssh_config
   delegate_to: localhost
 
-- name: Set fact for whether borg ssh config exists
-  ansible.builtin.set_fact:
-    ssh_config_defined: "{{ ssh_config.stat.exists }}"
+- name: Configure borgmatic and borg backups
+  when:
+    - ssh_config.stat.exists
+    - borg_repository is defined
+    - borg_encryption_passphrase is defined
+    - borg_remote_path is defined
+  block:
+    - name: Synchronize borgmatic files to remote
+      ansible.posix.synchronize:
+        src: "{{ role_path }}/files/"
+        dest: "{{ ansible_user_home_dir }}/borgmatic"
 
-- name: Sync local ssh config vault remote
-  when: ssh_config_defined
-  become: true
-  ansible.builtin.copy:
-    src: "{{ playbook_dir }}/vaults/borg/.ssh/"
-    dest: "{{ ansible_user_home_dir }}/borgmatic/.ssh"
-    mode: "0700"
-    owner: root
-    group: root
+    - name: Sync local ssh config vault remote
+      when: ssh_config_defined
+      become: true
+      ansible.builtin.copy:
+        src: "{{ playbook_dir }}/vaults/borg/.ssh/"
+        dest: "{{ ansible_user_home_dir }}/borgmatic/.ssh"
+        mode: "0700"
+        owner: root
+        group: root
 
-- name: Render Borgmatic configuration
-  become: true
-  ansible.builtin.template:
-    src: config.yaml.j2
-    dest: "{{ ansible_user_home_dir }}/borgmatic/config.yaml"
-    mode: "0600"
-    owner: "{{ ansible_ssh_user }}"
+    - name: Render Borgmatic configuration
+      become: true
+      ansible.builtin.template:
+        src: config.yaml.j2
+        dest: "{{ ansible_user_home_dir }}/borgmatic/config.yaml"
+        mode: "0600"
+        owner: "{{ ansible_ssh_user }}"
 
-  vars:
-    borg_exclude_patterns:
-      - /data/backups
-    borg_exclude_from: []
-    borg_install_method: package
-    borg_user: "{{ ansible_user }}"
-    borg_source_directories:
-      - /data
-    borgmatic_hooks:
-      before_backup:
-        - echo "`date` - Starting backup."
-      mongodb_databases:
-        - name: all
-          hostname: datalab-database-1
-          port: 27017
-    borgmatic_timer: cron
-    borg_retention_policy:
-      keep_daily: 30
-      keep_weekly: 0
-      keep_monthly: 12
-      keep_yearly: 4
-    borg_one_file_system: true
-    borgmatic_store_atime: true
-    borgmatic_store_ctime: true
-    borg_encryption_passcommand: false
-    borg_remote_rate_limit: 0
-    borg_ssh_command: ssh
-    borg_lock_wait_time: 5
+      vars:
+        borg_exclude_patterns:
+          - /data/backups
+        borg_exclude_from: []
+        borg_install_method: package
+        borg_user: "{{ ansible_user }}"
+        borg_source_directories:
+          - /data
+        borgmatic_hooks:
+          before_backup:
+            - echo "`date` - Starting backup."
+          mongodb_databases:
+            - name: all
+              hostname: datalab-database-1
+              port: 27017
+        borgmatic_timer: cron
+        borg_retention_policy:
+          keep_daily: 30
+          keep_weekly: 0
+          keep_monthly: 12
+          keep_yearly: 4
+        borg_one_file_system: true
+        borgmatic_store_atime: true
+        borgmatic_store_ctime: true
+        borg_encryption_passcommand: false
+        borg_remote_rate_limit: 0
+        borg_ssh_command: ssh
+        borg_lock_wait_time: 5
 
-- name: Build borgmatic image
-  become: true
-  community.docker.docker_image:
-    name: datalab-borgmatic
-    source: build
-    state: present
-    force_source: true
-    build:
-      path: "{{ ansible_user_home_dir }}/borgmatic"
+    - name: Build borgmatic image
+      become: true
+      community.docker.docker_image:
+        name: datalab-borgmatic
+        source: build
+        state: present
+        force_source: true
+        build:
+          path: "{{ ansible_user_home_dir }}/borgmatic"
 
-- name: Create borg repository if it does not exist
-  ansible.builtin.shell:
-    cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic borgmatic init --encryption=repokey -c /etc/borgmatic/config.yaml
-    executable: /bin/bash
-  register: new_borg_repository
-  changed_when: '"Repository already exists" not in new_borg_repository.stdout'
-  failed_when: new_borg_repository.rc != 0
+    - name: Create borg repository if it does not exist
+      ansible.builtin.shell:
+        cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic borgmatic init --encryption=repokey -c /etc/borgmatic/config.yaml
+        executable: /bin/bash
+      register: new_borg_repository
+      changed_when: '"Repository already exists" not in new_borg_repository.stdout'
+      failed_when: new_borg_repository.rc != 0
 
-- name: Perform first backup if borg repo was just created
-  ansible.builtin.shell:
-    cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: no-changed-when
-    executable: /bin/bash
-  when: new_borg_repository.changed # noqa: no-handler
+    - name: Perform first backup if borg repo was just created
+      ansible.builtin.shell:
+        cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: no-changed-when
+        executable: /bin/bash
+      when: new_borg_repository.changed # noqa: no-handler
 
-- name: Add Cron job for borgmatic
-  ansible.builtin.cron:
-    name: borgmatic
-    hour: "2"
-    minute: "{{ range(0, 59) | random(seed=inventory_hostname) }}"
-    user: "{{ ansible_user }}"
-    job: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: line-length
+    - name: Add Cron job for borgmatic
+      ansible.builtin.cron:
+        name: borgmatic
+        hour: "2"
+        minute: "{{ range(0, 59) | random(seed=inventory_hostname) }}"
+        user: "{{ ansible_user }}"
+        job: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: line-length
diff --git a/ansible/roles/monitoring/tasks/main.yml b/ansible/roles/monitoring/tasks/main.yml
@@ -0,0 +1,85 @@
+---
+- name: Check if prometheus_url is configured in inventory
+  when:
+    - prometheus_remote_write_url is defined
+    - prometheus_user is defined
+    - prometheus_password is defined
+  block:
+    - name: Launch node_exporter container
+      community.docker.docker_container:
+        name: node-exporter
+        image: prom/node-exporter:v1.9.1
+        network_mode: host
+        pid_mode: host
+        state: started
+        restart_policy: always
+        command:
+          - --path.rootfs=/host
+          - --path.procfs=/host/proc
+          - --path.sysfs=/host/sys
+          - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)
+        ports:
+          - 9100:9100
+        volumes:
+          # Mount the host's /proc directory to the container's /host/proc directory
+          # This is necessary for the node_exporter to be able to access the host's metrics
+          - /:/host:ro,rslave
+          - /proc:/host/proc:ro
+          - /sys:/host/sys:ro
+          - /etc/machine-id:/etc/machine-id:ro
+          - /etc/timezone:/etc/timezone:ro
+        healthcheck:
+          test: [CMD, wget, -q, --spider, http://localhost:9100/metrics]
+          interval: 30s
+          timeout: 10s
+          retries: 3
+          start_period: 5s
+
+    - name: Launch cadvisor container
+      community.docker.docker_container:
+        name: cadvisor
+        image: gcr.io/cadvisor/cadvisor:v0.52.1
+        network_mode: host
+        pid_mode: host
+        state: started
+        restart_policy: always
+        ports:
+          - 8080:8080
+        volumes:
+          - /:/rootfs:ro
+          - /var/run:/var/run:rw
+          - /sys:/sys:ro
+          - /var/lib/docker/:/var/lib/docker:ro
+
+    - name: Render prometheus config
+      ansible.builtin.template:
+        src: prometheus.yml.j2
+        dest: "{{ ansible_user_home_dir }}/prometheus.yml"
+        mode: "0644"
+      register: prometheus_config
+
+    - name: Launch prometheus container
+      community.docker.docker_container:
+        name: prometheus
+        image: prom/prometheus:v3.6.0
+        recreate: "{{ prometheus_config.changed }}"
+        network_mode: host
+        state: started
+        restart_policy: always
+        command:
+          - --config.file=/etc/prometheus/prometheus.yml
+          - --storage.tsdb.path=/prometheus
+          - --web.console.libraries=/etc/prometheus/console_libraries
+          - --web.console.templates=/etc/prometheus/consoles
+          - --web.enable-lifecycle
+        ports:
+          - 9090:9090
+        volumes:
+          - "prometheus_data:/prometheus"
+          - "{{ ansible_user_home_dir }}/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
+        healthcheck:
+          test: [CMD, wget, -q, --spider, http://localhost:9090/-/healthy]
+          interval: 30s
+          timeout: 10s
+          retries: 3
+          start_period: 5s
diff --git a/ansible/roles/monitoring/templates/prometheus.yml.j2 b/ansible/roles/monitoring/templates/prometheus.yml.j2
@@ -0,0 +1,28 @@
+global:
+  scrape_interval: 1m
+  scrape_timeout: 45s
+
+  external_labels:
+    instance: {{ app_url }}
+    environment: production
+    app: datalab
+    datalab: {{ datalab_prefix | default(app_url) }}
+
+scrape_configs:
+  - job_name: 'node-exporter'
+    static_configs:
+      - targets: ['localhost:9100']
+        labels:
+          service: system
+
+  - job_name: 'cadvisor'
+    static_configs:
+      - targets: ['localhost:8080']
+        labels:
+          service: docker
+
+remote_write:
+  - url: {{ prometheus_remote_write_url }}
+    basic_auth:
+      username: {{ prometheus_user }}
+      password: {{ prometheus_password }}
diff --git a/sync-ansible-upstream.sh b/sync-ansible-upstream.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e -u -o pipefail
 commit=$(cd src/datalab-ansible-terraform && git describe --tags)
-rsync --exclude vaults --exclude inventory.yml -avr src/datalab-ansible-terraform/ansible .
+rsync --exclude vaults --exclude inventory.yml -avr src/datalab-ansible-terraform/sync-ansible-upstream.sh src/datalab-ansible-terraform/Makefile src/datalab-ansible-terraform/.vault-pass.sh src/datalab-ansible-terraform/README.md src/datalab-ansible-terraform/requirements.txt src/datalab-ansible-terraform/ansible .
 git add -p ansible
 git add src/datalab-ansible-terraform
 git add $(git ls-files ansible --others --exclude-standard)
