Merge pull request #1311 from HarHarLinks/master

add auto proxy synapse worker metrics

docs/configuring-playbook-prometheus-grafana.md

@@ -58,6 +58,38 @@ Name | Description
 `matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable)
 `matrix_server_fqn_grafana`|Use this variable to override the domain at which the Grafana web user-interface is at (defaults to `stats.DOMAIN`)
 
+### Collecting worker metrics to an external Prometheus server
+
+If you are using workers (`matrix_synapse_workers_enabled`) and have enabled `matrix_nginx_proxy_proxy_synapse_metrics` as described above, the playbook will also automatically proxy the all worker threads's metrics to `https://matrix.DOMAIN/_synapse-worker-TYPE-ID/metrics`, where `TYPE` corresponds to the type and `ID` to the instanceId of a worker as exemplified in `matrix_synapse_workers_enabled_list`.
+
+The playbook also generates an exemplary prometheus.yml config file (`matrix_base_data_path/external_prometheus.yml.template`) with all the correct paths which you can copy to your Prometheus server and adapt to your needs, especially edit the specified `password_file` path and contents and path to your `synapse-v2.rules`.
+It will look a bit like this:
+```yaml
+scrape_configs:
+  - job_name: 'synapse'
+    metrics_path: /_synapse/metrics
+    scheme: https
+    basic_auth:
+      username: prometheus
+      password_file: /etc/prometheus/password.pwd
+    static_configs:
+      - targets: ['matrix.DOMAIN:443']
+        labels:
+          job: "master"
+          index: 1
+  - job_name: 'synapse-generic_worker-1'
+    metrics_path: /_synapse-worker-generic_worker-18111/metrics
+    scheme: https
+    basic_auth:
+      username: prometheus
+      password_file: /etc/prometheus/password.pwd
+    static_configs:
+      - targets: ['matrix.DOMAIN:443']
+        labels:
+          job: "generic_worker"
+          index: 18111
+```
+
 ### Collecting system and Postgres metrics to an external Prometheus server (advanced)
 
 When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats.

group_vars/matrix_servers

@@ -1339,6 +1339,8 @@ matrix_nginx_proxy_synapse_media_repository_locations: "{{matrix_synapse_workers
 matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}"
 matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}"
 
+matrix_nginx_proxy_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}"
+
 matrix_nginx_proxy_systemd_wanted_services_list: |
   {{
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])

roles/matrix-nginx-proxy/defaults/main.yml

@@ -182,6 +182,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809
 
 # Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain)
 matrix_nginx_proxy_proxy_synapse_metrics: false
+matrix_nginx_proxy_synapse_workers_enabled_list: []
 matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false
 # The following value will be written verbatim to the htpasswd file that stores the password for nginx to check against and needs to be encoded appropriately.
 # Read the manpage at `man 1 htpasswd` to learn more, then encrypt your password, and paste the encrypted value here.
@@ -226,7 +227,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: |
     +
     (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else [])
     +
-    (['/_synapse/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
+    (['/_synapse.*/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
   }}
 
 # Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected.

roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml

@@ -38,6 +38,15 @@
     mode: 0400
   when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool"
 
+- name: Generate sample prometheus.yml for external scraping
+  template:
+    src: "{{ role_path }}/templates/prometheus/external_prometheus.yml.example.j2"
+    dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: 0644
+  when: matrix_nginx_proxy_proxy_synapse_metrics|bool
+
 - name: Ensure Matrix nginx-proxy configured (generic)
   template:
     src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2"
@@ -270,3 +279,9 @@
     path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd"
     state: absent
   when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool"
+
+- name: Ensure sample prometheus.yml for external scraping is deleted
+  file:
+    path: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
+    state: absent
+  when: "not matrix_nginx_proxy_proxy_synapse_metrics|bool"

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2

@@ -153,6 +153,24 @@ server {
 	}
 	{% endif %}
 
+	{% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %}
+		{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
+			{% if worker.metrics_port != 0 %}
+				location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics {
+					resolver 127.0.0.11 valid=5s;
+					set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}";
+					proxy_pass http://$backend/_synapse/metrics;
+					proxy_set_header Host $host;
+
+					{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+						auth_basic "protected";
+						auth_basic_user_file /nginx-data/matrix-synapse-metrics-htpasswd;
+					{% endif %}
+				}
+			{% endif %}
+		{% endfor %}
+	{% endif %}
+
 	{# Everything else just goes to the API server ##}
 	location / {
 		{% if matrix_nginx_proxy_enabled %}

roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2

@@ -0,0 +1,40 @@
+global:
+  scrape_interval: 5s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'synapse-{{ matrix_domain }}'
+
+rule_files:
+  - /etc/prometheus/synapse-v2.rules
+
+scrape_configs:
+  - job_name: 'synapse'
+    metrics_path: /_synapse/metrics
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
+{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
+        labels:
+          job: "master"
+          index: 1
+{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
+  - job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}'
+    metrics_path: /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
+{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
+        labels:
+          job: "{{ worker.type }}"
+          index: {{ worker.instanceId }}
+{% endfor %}