Prometheus

Initial attempt. Seems to work fine.

Only tested on debian amd64 so far

group_vars/matrix_servers

@@ -1368,6 +1368,27 @@ matrix_synapse_admin_container_self_build: "{{ matrix_architecture != 'amd64' }}
 
 
 
+######################################################################
+#
+# matrix-prometheus
+#
+######################################################################
+
+matrix_prometheus_enabled: false
+
+# Normally, matrix-nginx-proxy is enabled and nginx can reach Prometheus over the container network.
+# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose
+# Prometheus' HTTP port to the local host.
+matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9090' }}"
+
+######################################################################
+#
+# /matrix-prometheus
+#
+######################################################################
+
+
+
 ######################################################################
 #
 # matrix-registration

roles/matrix-prometheus/defaults/main.yml

@@ -0,0 +1,51 @@
+# matrix-prometheus is an open-source systems monitoring and alerting toolkit
+# See: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
+
+matrix_prometheus_enabled: false
+
+matrix_prometheus_docker_image: "docker.io/prom/prometheus:v2.24.1"
+matrix_prometheus_docker_image_force_pull: "{{ matrix_prometheus_docker_image.endswith(':latest') }}"
+
+matrix_synapse_prometheus_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_synapse_docker_image_tag }}/contrib/prometheus/synapse-v2.rules"
+
+matrix_prometheus_base_path: "{{ matrix_base_data_path }}/prometheus"
+matrix_prometheus_config_path: "{{ matrix_prometheus_base_path }}/config"
+matrix_prometheus_data_path: "{{ matrix_prometheus_base_path }}/data"
+
+# A list of extra arguments to pass to the container
+matrix_prometheus_container_extra_arguments: []
+
+# List of systemd services that matrix-prometheus.service depends on
+matrix_prometheus_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-prometheus.service wants
+matrix_prometheus_systemd_wanted_services_list: []
+
+# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9090 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9090"), or empty string to not expose.
+matrix_prometheus_container_http_host_bind_port: ''
+
+
+# Default prometheus configuration template which covers the generic use case.
+# You can customize it by controlling the various variables inside it.
+#
+# For a more advanced customization, you can extend the default (see `matrix_prometheus_configuration_extension_yaml`)
+# or completely replace this variable with your own template.
+matrix_prometheus_configuration_yaml: "{{ lookup('template', 'templates/prometheus.yml.j2') }}"
+
+matrix_prometheus_configuration_extension_yaml: |
+  # Your custom YAML configuration goes here.
+  # This configuration extends the default starting configuration (`matrix_prometheus_configuration_yaml`).
+  #
+  # You can override individual variables from the default configuration, or introduce new ones.
+  #
+  # If you need something more special, you can take full control by
+  # completely redefining `matrix_prometheus_configuration_yaml`.
+
+matrix_prometheus_configuration_extension: "{{ matrix_prometheus_configuration_extension_yaml|from_yaml if matrix_prometheus_configuration_extension_yaml|from_yaml is mapping else {} }}"
+
+# Holds the final configuration (a combination of the default and its extension).
+# You most likely don't need to touch this variable. Instead, see `matrix_prometheus_configuration_yaml`.
+matrix_prometheus_configuration: "{{ matrix_prometheus_configuration_yaml|from_yaml|combine(matrix_prometheus_configuration_extension, recursive=True) }}"
+

roles/matrix-prometheus/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus.service'] }}"
+  when: matrix_prometheus_enabled|bool
+
+

roles/matrix-prometheus/tasks/main.yml

@@ -0,0 +1,14 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: run_setup|bool
+  tags:
+    - setup-all
+    - setup-prometheus
+
+- import_tasks: "{{ role_path }}/tasks/setup.yml"
+  tags:
+    - setup-all
+    - setup-prometheus

roles/matrix-prometheus/tasks/setup.yml

@@ -0,0 +1,93 @@
+---
+
+#
+# Tasks related to setting up matrix-prometheus
+#
+
+- name: Ensure matrix-prometheus image is pulled
+  docker_image:
+    name: "{{ matrix_prometheus_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_docker_image_force_pull }}"
+  when: "matrix_prometheus_enabled|bool"
+
+- name: Ensure Prometheus paths exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - "{{ matrix_prometheus_base_path }}"
+    - "{{ matrix_prometheus_config_path }}"
+    - "{{ matrix_prometheus_data_path }}"
+  when: matrix_prometheus_enabled|bool
+  
+- name: Ensure prometheus.yml installed
+  copy:
+    content: "{{ matrix_prometheus_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_prometheus_config_path }}/prometheus.yml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  when: matrix_prometheus_enabled|bool
+
+- name: Download synapse-v2.rules
+  get_url:
+    url: "{{ matrix_synapse_prometheus_rules_download_url }}"
+    dest: "{{ matrix_prometheus_config_path }}/synapse-v2.rules"
+    force: true
+    mode: 0440
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  when: matrix_prometheus_enabled|bool
+
+
+- name: Ensure matrix-prometheus.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-prometheus.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-prometheus.service"
+    mode: 0644
+  register: matrix_prometheus_systemd_service_result
+  when: matrix_prometheus_enabled|bool
+
+- name: Ensure systemd reloaded after matrix-prometheus.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_enabled|bool and matrix_prometheus_systemd_service_result.changed"
+
+#
+# Tasks related to getting rid of matrix-prometheus (if it was previously enabled)
+#
+
+- name: Check existence of matrix-prometheus service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus.service"
+  register: matrix_prometheus_service_stat
+
+- name: Ensure matrix-prometheus is stopped
+  service:
+    name: matrix-prometheus
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus.service"
+    state: absent
+  when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-prometheus.service removal
+  service:
+    daemon_reload: yes
+  when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_prometheus_docker_image }}"
+    state: absent
+  when: "not matrix_prometheus_enabled|bool"

roles/matrix-prometheus/tasks/validate_config.yml

@@ -0,0 +1,7 @@
+---
+
+- name: Fail if Synapse metrics not enabled
+  fail:
+    msg: >
+      You need to enable `matrix_synapse_metrics_enabled` for Prometheus grab metrics.
+  when: "not matrix_synapse_metrics_enabled"

roles/matrix-prometheus/templates/prometheus.yml.j2

@@ -0,0 +1,35 @@
+#jinja2: lstrip_blocks: "True"
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+    {% if matrix_synapse_metrics_enabled %}
+    - 'synapse-v2.rules'
+    {% endif %}
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+    scrape_timeout: 5s
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['localhost:9090']
+
+  {% if matrix_synapse_metrics_enabled %}
+  - job_name: 'synapse'
+    metrics_path: '/_synapse/metrics'
+    static_configs:
+    - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}']
+  {% endif %}
+

roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2

@@ -0,0 +1,42 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-prometheus
+{% for service in matrix_prometheus_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_prometheus_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_prometheus_container_http_host_bind_port %}
+			-p {{ matrix_prometheus_container_http_host_bind_port }}:9090 \
+			{% endif %}
+			-v {{ matrix_prometheus_config_path }}:/etc/prometheus:z \
+			-v {{ matrix_prometheus_data_path }}:/prometheus:z \
+			{% for arg in matrix_prometheus_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_prometheus_docker_image }}
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-prometheus
+
+[Install]
+WantedBy=multi-user.target

setup.yml

@@ -28,6 +28,7 @@
     - matrix-bot-matrix-reminder-bot
     - matrix-synapse
     - matrix-synapse-admin
+    - matrix-prometheus
     - matrix-registration
     - matrix-client-element
     - matrix-jitsi