Merge branch 'synapse-workers' into feature/add-worker-support

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2

@@ -101,6 +101,114 @@
 	}
 	{% endif %}
 
+	{% if synchrotron_workers %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #}
+	location /_matrix/client/r0/sync {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	location /_matrix/client/r0/events {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	location /_matrix/client/r0/initialSync {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ {
+		proxy_pass http://synchrotron$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
+	{% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %}
+	{% if client_reader_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
+	location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ client_reader_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
+	{% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %}
+	{% if media_repository_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
+	location /_matrix/media/ {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ media_repository_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
+	location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ media_repository_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
+	{% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %}
+	{% if event_creator_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
+	location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ event_creator_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
+	{% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %}
+	{% if frontend_proxy_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
+	location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% if not matrix_synapse_use_presence %}
+		location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver 127.0.0.11 valid=5s;
+			set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
+			proxy_pass http://$backend$request_uri;
+			proxy_set_header Host $host;
+			proxy_set_header X-Forwarded-For $remote_addr;
+		}
+	{% endif %}
+	{% endif %}
+
+	{% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %}
+	{% if user_dir_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
+	location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ user_dir_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
 	{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
 		{{- configuration_block }}
 	{% endfor %}
@@ -174,6 +282,19 @@
 	}
 {% endmacro %}
 
+{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %}
+{% if synchrotron_workers %}
+upstream synchrotron {
+	# ensures that requests from the same client will always be passed
+	# to the same server (except when this server is unavailable)
+	ip_hash;
+
+	{% for synchrotron_worker in synchrotron_workers %}
+	server "matrix-synapse:{{ synchrotron_worker.port }}";
+	{% endfor %}
+}
+{% endif %}
+
 server {
 	listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
 	server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }};
@@ -255,6 +376,19 @@ server {
 		ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
 	{% endif %}
 
+	{% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %}
+	{% if federation_reader_worker %}
+	{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #}
+	location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
+		{# Use the embedded DNS resolver in Docker containers to discover the service #}
+		resolver 127.0.0.11 valid=5s;
+		set $backend "matrix-synapse:{{ federation_reader_worker.port }}";
+		proxy_pass http://$backend$request_uri;
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+	{% endif %}
+
 	location / {
 		{% if matrix_nginx_proxy_enabled %}
 			{# Use the embedded DNS resolver in Docker containers to discover the service #}

roles/matrix-synapse/defaults/main.yml

@@ -260,6 +260,43 @@ matrix_synapse_metrics_port: 9100
 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md
 matrix_synapse_manhole_enabled: false
 
+# Enable support for Synapse workers
+matrix_synapse_workers_enabled: false
+
+# List of workers to spawn
+matrix_synapse_workers_enabled_list: []
+
+# Default list of workers to spawn
+matrix_synapse_workers_enabled_list:
+  - { worker: synchrotron, port: 18082 }
+  - { worker: synchrotron, port: 18083 }
+  - { worker: synchrotron, port: 18084 }
+  - { worker: appservice, port: 18085 }
+  - { worker: client_reader, port: 18086 }
+  - { worker: event_creator, port: 18087 }
+  - { worker: federation_reader, port: 18088 }
+  - { worker: federation_sender, port: 18089 }
+  - { worker: frontend_proxy, port: 18090 }
+  - { worker: media_repository, port: 18091 }
+  - { worker: pusher, port: 18092 }
+  - { worker: user_dir, port: 18093 }
+
+# The list of available workers (2020-04-14)
+matrix_synapse_workers_avail_list:
+  - appservice
+  - client_reader
+  - event_creator
+  - federation_reader
+  - federation_sender
+  - frontend_proxy
+  - media_repository
+  - pusher
+  - synchrotron
+  - user_dir
+
+# Ports used for communication between main synapse process and workers
+matrix_synapse_replication_tcp_port: 9092
+matrix_synapse_replication_http_port: 9093
 
 # Send ERROR logs to sentry.io for easier tracking
 # To set this up: go to sentry.io, create a python project, and set

roles/matrix-synapse/tasks/setup_synapse.yml

@@ -18,6 +18,8 @@
 
 - import_tasks: "{{ role_path }}/tasks/ext/setup.yml"
 
+- import_tasks: "{{ role_path }}/tasks/workers/setup.yml"
+
 - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml"
 
 - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml"

roles/matrix-synapse/tasks/workers/setup.yml

@@ -0,0 +1,8 @@
+---
+# a negative when condition will not actually prevent ansible from executing loops in imported tasks!
+
+- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
+  when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
+
+- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml"
+  when: "not matrix_synapse_workers_enabled|bool"

roles/matrix-synapse/tasks/workers/setup_install.yml

@@ -0,0 +1,42 @@
+---
+
+- name: Ensure synapse worker base service file installed
+  template:
+    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
+    mode: 0644
+  register: matrix_synapse_worker_systemd_service_result
+
+- name: Ensure previous worker service symlinks are cleaned (FIXME)
+  file:
+    path: "{{ item.root + '/' + item.path }}"
+    state: absent
+  when:
+    - matrix_synapse_workers_enabled|bool
+    - item.state == 'link'
+    - item.path is match('matrix-synapse-worker@.*\\.service')
+  with_filetree:
+    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+
+- name: Ensure systemd reloaded the worker service unit
+  service:
+    daemon_reload: yes
+
+- name: Ensure individual worker service symlinks exist
+  service:
+    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    enabled: true
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+
+- name: Ensure creation of specific worker configs
+  template:
+    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
+    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
+  with_list: "{{ matrix_synapse_workers_enabled_list }}"
+
+- name: Add workers to synapse.wants list
+  set_fact:
+    matrix_synapse_systemd_wanted_services_list: >
+      {{ matrix_synapse_systemd_wanted_services_list +
+      ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"

roles/matrix-synapse/tasks/workers/setup_uninstall.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Populate service facts
+  service_facts:
+
+- name: Ensure any worker services are stopped
+  service:
+    name: "{{ item.key }}"
+    state: stopped
+  with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
+
+# As we cannot know the ports of workers removed from the enabled_list..
+# => .. just kill them all (FIXME?)
+- name: Ensure previous worker service symlinks are cleaned
+  file:
+    path: "{{ item.root + '/' + item.path }}"
+    state: absent
+  when:
+    - not matrix_synapse_workers_enabled|bool
+    - item.state == 'link'
+    - item.path is match('matrix-synapse-worker@.*\\.service')
+  with_filetree:
+    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+
+- name: Ensure synapse worker base service file gets removed
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
+    state: absent
+  register: matrix_synapse_worker_systemd_service_result
+
+- name: Remove workers from synapse.wants list
+  set_fact:
+    matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}"
+  with_items: "{{ matrix_synapse_workers_avail_list }}"
+
+- name: Ensure systemd noticed removal of worker service units
+  service:
+    daemon_reload: yes

roles/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -223,6 +223,44 @@ listeners:
     type: manhole
 {% endif %}
 
+{% if matrix_synapse_workers_enabled %}
+  # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md
+  # TCP replication: streaming data from the master to the workers
+  - port: {{ matrix_synapse_replication_tcp_port }}
+    bind_addresses: ['0.0.0.0']
+    type: replication
+
+  # HTTP replication: for the workers to send data to the main synapse process
+  - port: {{ matrix_synapse_replication_http_port }}
+    bind_addresses: ['0.0.0.0']
+    type: http
+    resources:
+      - names: [replication]
+
+# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md
+worker_app: synapse.app.homeserver
+
+# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
+# reduce the main worker's offerings to core homeserver business
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list  %}
+notify_appservices: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list  %}
+send_federation: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list  %}
+enable_media_repo: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list  %}
+start_pushers: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list  %}
+update_user_directory: false
+{% endif %}
+
+# rather let systemd handle the forking
+daemonize: false
+{% endif %}
 
 # Forward extremities can build up in a room due to networking delays between
 # homeservers. Once this happens in a large room, calculation of the state of

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2

@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+# c.f. https://github.com/matrix-org/synapse/pull/4662
+[Unit]
+Description=Synapse Matrix Worker
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+Type=simple
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre=/bin/sleep 5
+
+# systemd ftw 🤦‍♂️
+# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
+ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
+			exec /usr/bin/docker exec \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			matrix-synapse \
+			python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
+
+ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i
+KillMode=process
+Restart=always
+RestartSec=10
+SyslogIdentifier=matrix-synapse-%i
+
+[Install]
+WantedBy=matrix-synapse.service

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2

@@ -43,6 +43,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
 			{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
 			-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
 			{% endif %}
+			{% if matrix_synapse_workers_enabled %}
+			{% for worker in matrix_synapse_workers_enabled_list %}
+			-p {{ worker.port }}:{{ worker.port }} \
+			{% endfor %}
+			{% endif %}
 			-v {{ matrix_synapse_config_dir_path }}:/data:ro \
 			-v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \
 			{% for volume in matrix_synapse_container_additional_volumes %}

roles/matrix-synapse/templates/synapse/worker.yaml.j2

@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+worker_app: synapse.app.{{ item.worker }}
+
+worker_replication_host: 127.0.0.1
+worker_replication_port: {{ matrix_synapse_replication_tcp_port }}
+worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
+
+{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
+worker_listeners:
+  - type: http
+    port: {{ item.port }}
+    resources:
+      - names:
+{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %}
+        - client
+{% elif item.worker in [ 'federation_reader' ] %}
+        - federation
+{% elif item.worker in [ 'media_repository' ] %}
+        - media
+{% endif %}
+{% endif %}
+
+{% if item.worker == 'frontend_proxy' %}
+worker_main_http_uri: http://127.0.0.1:8008
+{% endif %}
+  
+worker_daemonize: false
+worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
+worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config