Merge pull request #3436 from Michael-Hollister/michael/mmr-metrics-proxying

Added MMR metrics proxying support

group_vars/matrix_servers

@@ -3611,6 +3611,12 @@ matrix_media_repo_container_labels_traefik_internal_media_entrypoints: "{{ matri
 matrix_media_repo_container_labels_traefik_internal_matrix_client_media_enabled: "{{ matrix_playbook_internal_matrix_client_api_traefik_entrypoint_enabled }}"
 matrix_media_repo_container_labels_traefik_internal_matrix_client_media_entrypoints: "{{ matrix_playbook_internal_matrix_client_api_traefik_entrypoint_name }}"
 
+matrix_media_repo_metrics_proxying_enabled: "{{ matrix_media_repo_metrics_enabled and matrix_metrics_exposure_enabled }}"
+matrix_media_repo_metrics_proxying_hostname: "{{ matrix_metrics_exposure_hostname }}"
+matrix_media_repo_metrics_proxying_path: "{{ matrix_metrics_exposure_path_prefix }}/matrix-media-repo"
+matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_enabled: "{{ matrix_metrics_exposure_http_basic_auth_enabled }}"
+matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_users: "{{ matrix_metrics_exposure_http_basic_auth_users }}"
+
 matrix_media_repo_database_hostname: "{{ devture_postgres_connection_hostname if devture_postgres_enabled else '' }}"
 matrix_media_repo_database_username: matrix_media_repo
 matrix_media_repo_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'mediarepo.db', rounds=655555) | to_uuid }}"
@@ -4880,11 +4886,6 @@ grafana_provisioning_dashboard_template_files: |
         'path': 'roles/custom/matrix-prometheus-nginxlog-exporter/templates/grafana/nginx-proxy.json',
         'name': 'nginx-proxy.json',
     }] if matrix_prometheus_nginxlog_exporter_enabled else [])
-    +
-    ([{
-        'path': 'roles/custom/matrix-media-repo/templates/grafana/media-repo.json',
-        'name': 'media-repo.json',
-    }] if matrix_media_repo_metrics_enabled else [])
   }}
 
 grafana_default_home_dashboard_path: |-

roles/custom/matrix-media-repo/defaults/main.yml

@@ -44,6 +44,11 @@ matrix_media_repo_container_network: "{{ matrix_media_repo_identifier }}"
 # Use this to expose this container to another reverse proxy, which runs in a different container network.
 matrix_media_repo_container_additional_networks: []
 
+# Controls whether media repo metrics should be proxied (exposed) on `matrix.DOMAIN/metrics/matrix-media-repo`
+matrix_media_repo_metrics_proxying_enabled: false
+matrix_media_repo_metrics_proxying_hostname: ""
+matrix_media_repo_metrics_proxying_path: "/metrics/matrix-media-repo"
+
 # Controls whether the matrix-media-repo container exposes its HTTP port (tcp/8000 in the container).
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:8000"), or empty string to not expose.
@@ -134,6 +139,18 @@ matrix_media_repo_container_labels_traefik_t2bot_entrypoints: "{{ matrix_media_r
 matrix_media_repo_container_labels_traefik_t2bot_tls: "{{ matrix_media_repo_container_labels_traefik_t2bot_entrypoints != 'web' }}"
 matrix_media_repo_container_labels_traefik_t2bot_tls_certResolver: default  # noqa var-naming
 
+# Controls whether labels will be added that expose the media repo metrics endpoint
+matrix_media_repo_container_labels_traefik_metrics_enabled: "{{ matrix_media_repo_metrics_enabled and matrix_media_repo_metrics_proxying_enabled }}"
+matrix_media_repo_container_labels_traefik_metrics_rule: "Host(`{{ matrix_media_repo_metrics_proxying_hostname }}`) && PathPrefix(`{{ matrix_media_repo_metrics_proxying_path }}`)"
+matrix_media_repo_container_labels_traefik_metrics_priority: 0
+matrix_media_repo_container_labels_traefik_metrics_entrypoints: "{{ matrix_media_repo_container_labels_traefik_entrypoints }}"
+matrix_media_repo_container_labels_traefik_metrics_tls: "{{ matrix_media_repo_container_labels_traefik_t2bot_entrypoints != 'web' }}"
+matrix_media_repo_container_labels_traefik_metrics_tls_certResolver: default  # noqa var-naming
+
+matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_enabled: false
+# See: https://doc.traefik.io/traefik/middlewares/http/basicauth/#users
+matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_users: ''
+
 # Traefik labels handling the old `/_matrix/media` endpoints on the federation entrypint.
 # These are being superseded by `/_matrix/federation/VERSION/media` endpoints - see `matrix_media_repo_container_labels_traefik_federation_matrix_federation_media_*`.
 matrix_media_repo_container_labels_traefik_media_federation_enabled: true

roles/custom/matrix-media-repo/templates/grafana/media-repo.json

@@ -1,59 +1,16 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_PROMETHEUS",
-      "label": "Prometheus",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
-  ],
-  "__elements": {},
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "10.1.0"
-    },
-    {
-      "type": "panel",
-      "id": "heatmap",
-      "name": "Heatmap",
-      "version": ""
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    },
-    {
-      "type": "panel",
-      "id": "timeseries",
-      "name": "Time series",
-      "version": ""
-    }
-  ],
   "annotations": {
     "list": [
       {
         "builtIn": 1,
         "datasource": {
-          "type": "datasource",
-          "uid": "grafana"
+          "type": "grafana",
+          "uid": "${DS_PROMETHEUS}"
         },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
         "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
         "type": "dashboard"
       }
     ]
@@ -153,7 +110,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(media_http_requests_total[2m])",
+          "expr": "rate(media_http_requests_total{host=\"$host\"}[2m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ '{{host}}: {{method}} {{action}}' }}",
@@ -166,7 +123,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(media_invalid_http_requests_total[2m])",
+          "expr": "rate(media_invalid_http_requests_total{host=\"$host\"}[2m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ 'Invalid Host: {{method}} {{action}}' }}",
@@ -265,7 +222,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(media_http_responses_total[2m])",
+          "expr": "rate(media_http_responses_total{host=\"$host\"}[2m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ '{{host}}: {{method}} {{action}} {{statusCode}}' }}",
@@ -278,7 +235,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "rate(media_invalid_http_requests_total[2m])",
+          "expr": "rate(media_invalid_http_requests_total{host=\"$host\"}[2m])",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,
@@ -369,7 +326,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(rate(media_http_response_time_seconds_bucket{action=~\"download|thumbnail\"}[2m])) by (le)",
+          "expr": "sum(rate(media_http_response_time_seconds_bucket{action=~\"download|thumbnail\", host=\"$host\"}[2m])) by (le)",
           "format": "heatmap",
           "instant": false,
           "interval": "",
@@ -460,7 +417,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(rate(media_http_response_time_seconds_bucket{action=\"upload\"}[2m])) by (le)",
+          "expr": "sum(rate(media_http_response_time_seconds_bucket{action=\"upload\", host=\"$host\"}[2m])) by (le)",
           "format": "heatmap",
           "instant": false,
           "legendFormat": "{{ '{{method}} {{action}} - {{le}}' }}",
@@ -560,7 +517,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "go_memstats_alloc_bytes{job=\"media_repo\"}",
+          "expr": "go_memstats_alloc_bytes",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -574,7 +531,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "go_memstats_sys_bytes{job=\"media_repo\"}",
+          "expr": "go_memstats_sys_bytes",
           "interval": "",
           "legendFormat": "memory usage (sys)",
           "refId": "C"
@@ -585,7 +542,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "go_memstats_heap_alloc_bytes{job=\"media_repo\"}",
+          "expr": "go_memstats_heap_alloc_bytes",
           "interval": "",
           "legendFormat": "heap usage (alloc)",
           "refId": "A"
@@ -596,7 +553,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "go_memstats_heap_idle_bytes{job=\"media_repo\"}",
+          "expr": "go_memstats_heap_idle_bytes",
           "interval": "",
           "legendFormat": "heap usage (idle)",
           "refId": "D"
@@ -607,7 +564,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "go_memstats_heap_inuse_bytes{job=\"media_repo\"}",
+          "expr": "go_memstats_heap_inuse_bytes",
           "interval": "",
           "legendFormat": "heap usage (used)",
           "refId": "E"
@@ -619,7 +576,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "go_memstats_heap_released_bytes{job=\"media_repo\"}",
+          "expr": "go_memstats_heap_released_bytes",
           "hide": false,
           "legendFormat": "heap usage (released)",
           "range": true,
@@ -718,7 +675,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "go_goroutines{job=\"media_repo\"}",
+          "expr": "go_goroutines",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -733,7 +690,7 @@
           },
           "editorMode": "code",
           "exemplar": true,
-          "expr": "go_threads{job=\"media_repo\"}",
+          "expr": "go_threads",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1263,9 +1220,51 @@
   "refresh": "1m",
   "schemaVersion": 38,
   "style": "dark",
-  "tags": [],
+  "tags": [
+    "matrix"
+  ],
   "templating": {
-    "list": []
+    "list": [
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
+        "definition": "label_values(media_http_response_time_seconds_bucket,host)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Host",
+        "multi": false,
+        "name": "host",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(media_http_response_time_seconds_bucket,host)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
   },
   "time": {
     "from": "now-1h",

roles/custom/matrix-media-repo/templates/media-repo/labels.j2

@@ -6,6 +6,7 @@ traefik.docker.network={{ matrix_media_repo_container_labels_traefik_docker_netw
 {% endif %}
 
 traefik.http.services.matrix-media-repo.loadbalancer.server.port={{ matrix_media_repo_port }}
+traefik.http.services.matrix-media-repo-metrics.loadbalancer.server.port={{ matrix_media_repo_metrics_port }}
 
 {% set middlewares = [] %}
 
@@ -243,6 +244,43 @@ traefik.http.routers.matrix-media-repo-public-t2bot.tls.certResolver={{ matrix_m
 {% endif %}
 
 
+{% if matrix_media_repo_container_labels_traefik_metrics_enabled %}
+############################################################
+#                                                          #
+# Metrics                                                  #
+#                                                          #
+############################################################
+
+{% set metricsMiddlewares = ['matrix-media-repo-metrics-replace-path'] %}
+traefik.http.middlewares.matrix-media-repo-metrics-replace-path.replacepath.path=/metrics
+
+{% if matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_enabled %}
+{% set metricsMiddlewares = metricsMiddlewares + ['matrix-media-repo-metrics-basic-auth'] %}
+traefik.http.middlewares.matrix-media-repo-metrics-basic-auth.basicauth.users={{ matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_users }}
+{% endif %}
+
+traefik.http.routers.matrix-media-repo-metrics.rule={{ matrix_media_repo_container_labels_traefik_metrics_rule }}
+traefik.http.routers.matrix-media-repo-metrics.middlewares={{ metricsMiddlewares | join(',') }}
+
+{% if matrix_media_repo_container_labels_traefik_metrics_priority | int > 0 %}
+traefik.http.routers.matrix-media-repo-metrics.priority={{ matrix_media_repo_container_labels_traefik_metrics_priority }}
+{% endif %}
+
+traefik.http.routers.matrix-media-repo-metrics.service=matrix-media-repo-metrics
+traefik.http.routers.matrix-media-repo-metrics.entrypoints={{ matrix_media_repo_container_labels_traefik_metrics_entrypoints }}
+
+traefik.http.routers.matrix-media-repo-metrics.tls={{ matrix_media_repo_container_labels_traefik_metrics_tls | to_json }}
+{% if matrix_media_repo_container_labels_traefik_metrics_tls %}
+traefik.http.routers.matrix-media-repo-metrics.tls.certResolver={{ matrix_media_repo_container_labels_traefik_metrics_tls_certResolver }}
+{% endif %}
+
+############################################################
+#                                                          #
+# /Metrics                                                 #
+#                                                          #
+############################################################
+{% endif %}
+
 {% if matrix_media_repo_homeserver_federation_enabled %}
 # Matrix Federation