From dfe8628fbfddf95e642d24ba7b4600bd8df78611 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 14 Mar 2026 02:56:48 +0200 Subject: [PATCH] matrix-synapse: add routing-focused reverse-proxy access log preset --- roles/custom/matrix-synapse/defaults/main.yml | 20 +++++++++++++ .../matrix-synapse/tasks/validate_config.yml | 7 +++++ ...ix-synapse-reverse-proxy-companion.conf.j2 | 30 ++++++++++++------- .../nginx/nginx.conf.j2 | 10 ++++--- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 6c7960747..f84401d26 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -1920,6 +1920,26 @@ matrix_synapse_reverse_proxy_companion_worker_connections: 1024 # Option to disable the access log matrix_synapse_reverse_proxy_companion_access_log_enabled: true +# Controls the regular nginx access log format used for `/var/log/nginx/access.log`. +# `routing_debug` is the default because it includes the chosen upstream label, +# the resolved backend address, and timing data, which makes it much easier to +# verify request routing in worker deployments. +# This does not affect the separate syslog integration format used by prometheus-nginxlog-exporter. +matrix_synapse_reverse_proxy_companion_access_log_format: routing_debug + +# The available values for `matrix_synapse_reverse_proxy_companion_access_log_format`. +# You can override this map to define custom formats, but that is fragile and discouraged. +matrix_synapse_reverse_proxy_companion_access_log_format_presets: + main: + - '$remote_addr - $remote_user [$time_local] "$request"' + - '$status $body_bytes_sent "$http_referer"' + - ' "$http_user_agent" "$http_x_forwarded_for"' + routing_debug: + - '$remote_addr - $remote_user [$time_local] "$request"' + - '$status $body_bytes_sent "$http_referer"' + - ' "$http_user_agent" "$http_x_forwarded_for"' + - ' "$host" "$matrix_upstream_label" "$upstream_addr" "$upstream_status" "$request_time" "$upstream_response_time"' + # Controls whether to send access logs to a remote syslog-compatible server matrix_synapse_reverse_proxy_companion_access_log_syslog_integration_enabled: false matrix_synapse_reverse_proxy_companion_access_log_syslog_integration_server_port: '' diff --git a/roles/custom/matrix-synapse/tasks/validate_config.yml b/roles/custom/matrix-synapse/tasks/validate_config.yml index 6ad3205d1..47df0834b 100644 --- a/roles/custom/matrix-synapse/tasks/validate_config.yml +++ b/roles/custom/matrix-synapse/tasks/validate_config.yml @@ -151,6 +151,13 @@ - "matrix_synapse_workers_stream_writer_presence_stream_workers_count" - "matrix_synapse_workers_stream_writer_push_rules_stream_workers_count" +- name: Fail if matrix-synapse-reverse-proxy-companion access log format is invalid + ansible.builtin.fail: + msg: >- + `matrix_synapse_reverse_proxy_companion_access_log_format` must be one of: + {{ matrix_synapse_reverse_proxy_companion_access_log_format_presets.keys() | sort | join(', ') }} + when: "matrix_synapse_reverse_proxy_companion_access_log_format not in matrix_synapse_reverse_proxy_companion_access_log_format_presets" + - name: Fail when mixing generic workers with new specialized workers ansible.builtin.fail: msg: >- diff --git a/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 index ce5ef6edc..080beb177 100644 --- a/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 +++ b/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 @@ -80,6 +80,7 @@ {% macro render_locations_to_upstream(locations, upstream_name) %} {% for location in locations %} location ~ {{ location }} { + set $matrix_upstream_label "{{ upstream_name }}"; proxy_pass http://{{ upstream_name }}$request_uri; proxy_http_version 1.1; proxy_set_header Connection ""; @@ -91,6 +92,7 @@ {% for location in locations %} location ~ {{ location }} { {% if workers | length > 0 %} + set $matrix_upstream_label "{{ upstream_name }}"; proxy_pass http://{{ upstream_name }}$request_uri; proxy_http_version 1.1; proxy_set_header Connection ""; @@ -107,6 +109,7 @@ {% macro render_locations_to_upstream_with_whoami_sync_worker_router(locations, upstream_name) %} {% for location in locations %} location ~ {{ location }} { + set $matrix_upstream_label "{{ upstream_name }}"; # Use auth_request to call the whoami sync worker router. # The handler resolves the access token to a user identifier and returns it # in the X-User-Identifier header, which is then used for upstream hashing. @@ -115,6 +118,7 @@ {% if matrix_synapse_reverse_proxy_companion_whoami_sync_worker_router_debug_headers_enabled %} add_header X-Sync-Worker-Router-User-Identifier $user_identifier always; + add_header X-Sync-Worker-Router-Upstream-Label $matrix_upstream_label always; add_header X-Sync-Worker-Router-Upstream $upstream_addr always; {% endif %} @@ -191,6 +195,7 @@ server { proxy_buffering on; proxy_max_temp_file_size 0; proxy_set_header Host $host; + set $matrix_upstream_label "synapse_main_client_api"; {% if matrix_synapse_reverse_proxy_companion_whoami_sync_worker_router_enabled %} # Internal location for whoami-based sync worker routing. @@ -266,13 +271,14 @@ server { {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }} {% endif %} - {% if media_repository_workers | length > 0 %} - # https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository - {% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %} - location ~ {{ location }} { - proxy_pass http://media_repository_workers_upstream$request_uri; + {% if media_repository_workers | length > 0 %} + # https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository + {% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %} + location ~ {{ location }} { + set $matrix_upstream_label "media_repository_workers_upstream"; + proxy_pass http://media_repository_workers_upstream$request_uri; - {% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %} + {% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %} proxy_cache {{ matrix_synapse_reverse_proxy_companion_synapse_cache_keys_zone_name }}; proxy_cache_valid any {{ matrix_synapse_reverse_proxy_companion_synapse_cache_proxy_cache_valid_time }}; proxy_force_ranges on; @@ -316,6 +322,7 @@ server { proxy_buffering on; proxy_max_temp_file_size 0; proxy_set_header Host $host; + set $matrix_upstream_label "synapse_main_federation_api"; {% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %} # Federation overrides — These locations must go to the main Synapse process @@ -337,11 +344,12 @@ server { {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations, 'generic_workers_upstream') }} {% endif %} - {% if media_repository_workers | length > 0 %} - # https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository - {% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %} - location ~ {{ location }} { - proxy_pass http://media_repository_workers_upstream$request_uri; + {% if media_repository_workers | length > 0 %} + # https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository + {% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %} + location ~ {{ location }} { + set $matrix_upstream_label "media_repository_workers_upstream"; + proxy_pass http://media_repository_workers_upstream$request_uri; {% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %} proxy_buffering on; diff --git a/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/nginx.conf.j2 b/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/nginx.conf.j2 index aa52a05e1..fa7499d54 100644 --- a/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/nginx.conf.j2 +++ b/roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/nginx.conf.j2 @@ -48,12 +48,14 @@ http { js_shared_dict_zone zone=whoami_sync_worker_router_cache:{{ matrix_synapse_reverse_proxy_companion_whoami_sync_worker_router_cache_size_mb }}m; {% endif %} - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; + {% set access_log_format_fragments = matrix_synapse_reverse_proxy_companion_access_log_format_presets[matrix_synapse_reverse_proxy_companion_access_log_format] %} + log_format access_log_fmt + {% for fragment in access_log_format_fragments %} + '{{ fragment }}'{% if loop.last %};{% endif %} + {% endfor %} {% if matrix_synapse_reverse_proxy_companion_access_log_enabled %} - access_log /var/log/nginx/access.log main; + access_log /var/log/nginx/access.log access_log_fmt; {% else %} access_log off; {% endif %}