matrix-synapse: add routing-focused reverse-proxy access log preset

roles/custom/matrix-synapse/defaults/main.yml

@@ -1920,6 +1920,26 @@ matrix_synapse_reverse_proxy_companion_worker_connections: 1024
 # Option to disable the access log
 matrix_synapse_reverse_proxy_companion_access_log_enabled: true
 
+# Controls the regular nginx access log format used for `/var/log/nginx/access.log`.
+# `routing_debug` is the default because it includes the chosen upstream label,
+# the resolved backend address, and timing data, which makes it much easier to
+# verify request routing in worker deployments.
+# This does not affect the separate syslog integration format used by prometheus-nginxlog-exporter.
+matrix_synapse_reverse_proxy_companion_access_log_format: routing_debug
+
+# The available values for `matrix_synapse_reverse_proxy_companion_access_log_format`.
+# You can override this map to define custom formats, but that is fragile and discouraged.
+matrix_synapse_reverse_proxy_companion_access_log_format_presets:
+  main:
+    - '$remote_addr - $remote_user [$time_local] "$request"'
+    - '$status $body_bytes_sent "$http_referer"'
+    - ' "$http_user_agent" "$http_x_forwarded_for"'
+  routing_debug:
+    - '$remote_addr - $remote_user [$time_local] "$request"'
+    - '$status $body_bytes_sent "$http_referer"'
+    - ' "$http_user_agent" "$http_x_forwarded_for"'
+    - ' "$host" "$matrix_upstream_label" "$upstream_addr" "$upstream_status" "$request_time" "$upstream_response_time"'
+
 # Controls whether to send access logs to a remote syslog-compatible server
 matrix_synapse_reverse_proxy_companion_access_log_syslog_integration_enabled: false
 matrix_synapse_reverse_proxy_companion_access_log_syslog_integration_server_port: ''

roles/custom/matrix-synapse/tasks/validate_config.yml

@@ -151,6 +151,13 @@
     - "matrix_synapse_workers_stream_writer_presence_stream_workers_count"
     - "matrix_synapse_workers_stream_writer_push_rules_stream_workers_count"
 
+- name: Fail if matrix-synapse-reverse-proxy-companion access log format is invalid
+  ansible.builtin.fail:
+    msg: >-
+      `matrix_synapse_reverse_proxy_companion_access_log_format` must be one of:
+      {{ matrix_synapse_reverse_proxy_companion_access_log_format_presets.keys() | sort | join(', ') }}
+  when: "matrix_synapse_reverse_proxy_companion_access_log_format not in matrix_synapse_reverse_proxy_companion_access_log_format_presets"
+
 - name: Fail when mixing generic workers with new specialized workers
   ansible.builtin.fail:
     msg: >-

roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2

@@ -80,6 +80,7 @@
 {% macro render_locations_to_upstream(locations, upstream_name) %}
 	{% for location in locations %}
 	location ~ {{ location }} {
+		set $matrix_upstream_label "{{ upstream_name }}";
 		proxy_pass http://{{ upstream_name }}$request_uri;
 		proxy_http_version 1.1;
 		proxy_set_header Connection "";
@@ -91,6 +92,7 @@
 	{% for location in locations %}
 	location ~ {{ location }} {
 		{% if workers | length > 0 %}
+		set $matrix_upstream_label "{{ upstream_name }}";
 		proxy_pass http://{{ upstream_name }}$request_uri;
 		proxy_http_version 1.1;
 		proxy_set_header Connection "";
@@ -107,6 +109,7 @@
 {% macro render_locations_to_upstream_with_whoami_sync_worker_router(locations, upstream_name) %}
 	{% for location in locations %}
 	location ~ {{ location }} {
+		set $matrix_upstream_label "{{ upstream_name }}";
 		# Use auth_request to call the whoami sync worker router.
 		# The handler resolves the access token to a user identifier and returns it
 		# in the X-User-Identifier header, which is then used for upstream hashing.
@@ -115,6 +118,7 @@
 
 		{% if matrix_synapse_reverse_proxy_companion_whoami_sync_worker_router_debug_headers_enabled %}
 		add_header X-Sync-Worker-Router-User-Identifier $user_identifier always;
+		add_header X-Sync-Worker-Router-Upstream-Label $matrix_upstream_label always;
 		add_header X-Sync-Worker-Router-Upstream $upstream_addr always;
 		{% endif %}
 
@@ -191,6 +195,7 @@ server {
 	proxy_buffering on;
 	proxy_max_temp_file_size 0;
 	proxy_set_header Host $host;
+	set $matrix_upstream_label "synapse_main_client_api";
 
 	{% if matrix_synapse_reverse_proxy_companion_whoami_sync_worker_router_enabled %}
 	# Internal location for whoami-based sync worker routing.
@@ -266,13 +271,14 @@ server {
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
 		{% endif %}
 
-		{% if media_repository_workers | length > 0 %}
-			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository
-			{% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %}
-			location ~ {{ location }} {
-				proxy_pass http://media_repository_workers_upstream$request_uri;
+			{% if media_repository_workers | length > 0 %}
+				# https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository
+				{% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %}
+				location ~ {{ location }} {
+					set $matrix_upstream_label "media_repository_workers_upstream";
+					proxy_pass http://media_repository_workers_upstream$request_uri;
 
-				{% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %}
+					{% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %}
 					proxy_cache            {{ matrix_synapse_reverse_proxy_companion_synapse_cache_keys_zone_name }};
 					proxy_cache_valid      any {{ matrix_synapse_reverse_proxy_companion_synapse_cache_proxy_cache_valid_time }};
 					proxy_force_ranges on;
@@ -316,6 +322,7 @@ server {
 	proxy_buffering on;
 	proxy_max_temp_file_size 0;
 	proxy_set_header Host $host;
+	set $matrix_upstream_label "synapse_main_federation_api";
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
 		# Federation overrides — These locations must go to the main Synapse process
@@ -337,11 +344,12 @@ server {
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations, 'generic_workers_upstream') }}
 		{% endif %}
 
-		{% if media_repository_workers | length > 0 %}
-			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository
-			{% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %}
-			location ~ {{ location }} {
-				proxy_pass http://media_repository_workers_upstream$request_uri;
+			{% if media_repository_workers | length > 0 %}
+				# https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository
+				{% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %}
+				location ~ {{ location }} {
+					set $matrix_upstream_label "media_repository_workers_upstream";
+					proxy_pass http://media_repository_workers_upstream$request_uri;
 
 				{% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %}
 					proxy_buffering        on;

roles/custom/matrix-synapse/templates/reverse_proxy_companion/nginx/nginx.conf.j2

@@ -48,12 +48,14 @@ http {
 	js_shared_dict_zone zone=whoami_sync_worker_router_cache:{{ matrix_synapse_reverse_proxy_companion_whoami_sync_worker_router_cache_size_mb }}m;
 	{% endif %}
 
-	log_format main '$remote_addr - $remote_user [$time_local] "$request" '
-		'$status $body_bytes_sent "$http_referer" '
-		'"$http_user_agent" "$http_x_forwarded_for"';
+	{% set access_log_format_fragments = matrix_synapse_reverse_proxy_companion_access_log_format_presets[matrix_synapse_reverse_proxy_companion_access_log_format] %}
+	log_format access_log_fmt
+	{% for fragment in access_log_format_fragments %}
+		'{{ fragment }}'{% if loop.last %};{% endif %}
+	{% endfor %}
 
 	{% if matrix_synapse_reverse_proxy_companion_access_log_enabled %}
-	access_log /var/log/nginx/access.log main;
+	access_log /var/log/nginx/access.log access_log_fmt;
 	{% else %}
 	access_log off;
 	{% endif %}