Add initial support for synapse workers

· needs documentation; no checks yet for port clashes or typos in worker name
· according to https://github.com/matrix-org/synapse/wiki/Workers-setup-with-nginx#results
  about 90% of requests go to the synchrotron endpoint
· thus, the synchrotron worker is especially suited to be load-balanced
· most of the other workers are documented to support only a single instance
· https://github.com/matrix-org/synapse/blob/master/docs/workers.md

roles/matrix-synapse/defaults/main.yml

@@ -258,6 +258,43 @@ matrix_synapse_metrics_port: 9100
 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md
 matrix_synapse_manhole_enabled: false
 
+# Enable support for Synapse workers
+matrix_synapse_workers_enabled: false
+
+# List of workers to spawn
+matrix_synapse_workers_enabled_list: []
+
+# Default list of workers to spawn
+matrix_synapse_workers_enabled_list:
+  - { worker: synchrotron, port: 8082 }
+  - { worker: synchrotron, port: 8083 }
+  - { worker: synchrotron, port: 8084 }
+  - { worker: appservice, port: 8085 }
+  - { worker: client_reader, port: 8086 }
+  - { worker: event_creator, port: 8087 }
+  - { worker: federation_reader, port: 8088 }
+  - { worker: federation_sender, port: 8089 }
+  - { worker: frontend_proxy, port: 8090 }
+  - { worker: media_repository, port: 8091 }
+  - { worker: pusher, port: 8092 }
+  - { worker: user_dir, port: 8093 }
+
+# The list of available workers (2020-04-14)
+matrix_synapse_workers_avail_list:
+  - appservice
+  - client_reader
+  - event_creator
+  - federation_reader
+  - federation_sender
+  - frontend_proxy
+  - media_repository
+  - pusher
+  - synchrotron
+  - user_dir
+
+# Ports used for communication between main synapse process and workers
+matrix_synapse_replication_tcp_port: 9092
+matrix_synapse_replication_http_port: 9093
 
 # Send ERROR logs to sentry.io for easier tracking
 # To set this up: go to sentry.io, create a python project, and set

roles/matrix-synapse/tasks/setup_synapse.yml

@@ -19,6 +19,8 @@
 
 - import_tasks: "{{ role_path }}/tasks/ext/setup.yml"
 
+- import_tasks: "{{ role_path }}/tasks/workers/setup.yml"
+
 - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml"
 
 - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml"

roles/matrix-synapse/tasks/workers/setup.yml

@@ -0,0 +1,7 @@
+---
+
+- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
+  when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
+
+- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml"
+  when: "not matrix_synapse_workers_enabled|bool"

roles/matrix-synapse/tasks/workers/setup_install.yml

@@ -0,0 +1,41 @@
+---
+
+- name: Ensure synapse worker base service file installed
+  template:
+    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
+    mode: 0644
+  register: matrix_synapse_worker_systemd_service_result
+
+- name: Ensure previous worker service symlinks are cleaned (FIXME)
+  file:
+    path: "{{ item.root + '/' + item.path }}"
+    state: absent
+  when:
+    - item.state == 'link'
+    - item.path is match('matrix-synapse-worker@*.service')
+  with_filetree:
+    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+
+- name: Ensure systemd reloaded the worker service unit
+  service:
+    daemon_reload: yes
+
+- name: Ensure individual worker service symlinks exist
+  service:
+    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    enabled: true
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+
+- name: Ensure creation of specific worker configs
+  template:
+    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
+    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
+  with_list: "{{ matrix_synapse_workers_enabled_list }}"
+
+- name: Add workers to synapse.wants list
+  set_fact:
+    matrix_synapse_systemd_wanted_services_list: >
+      {{ matrix_synapse_systemd_wanted_services_list +
+      ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"

roles/matrix-synapse/tasks/workers/setup_uninstall.yml

@@ -0,0 +1,34 @@
+---
+
+- name: Ensure individual worker services are stopped
+  service:
+    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    state: stopped
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+
+# As we cannot know the ports of workers removed from the enabled_list..
+# => .. just kill them all (FIXME?)
+- name: Ensure previous worker service symlinks are cleaned
+  file:
+    path: "{{ item.root + '/' + item.path }}"
+    state: absent
+  when:
+    - item.state == 'link'
+    - item.path is match('matrix-synapse-worker@*.service')
+  with_filetree:
+    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+
+- name: Ensure synapse worker base service file gets removed
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
+    state: absent
+  register: matrix_synapse_worker_systemd_service_result
+
+- name: Remove workers from synapse.wants list
+  set_fact:
+    matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}"
+  with_items: "{{ matrix_synapse_workers_avail_list }}"
+
+- name: Ensure systemd noticed removal of worker service units
+  service:
+    daemon_reload: yes

roles/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -251,6 +251,44 @@ listeners:
     type: manhole
 {% endif %}
 
+{% if matrix_synapse_workers_enabled %}
+  # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md
+  # TCP replication: streaming data from the master to the workers
+  - port: {{ matrix_synapse_replication_tcp_port }}
+    bind_addresses: ['0.0.0.0']
+    type: replication
+
+  # HTTP replication: for the workers to send data to the main synapse process
+  - port: {{ matrix_synapse_replication_http_port }}
+    bind_addresses: ['0.0.0.0']
+    type: http
+    resources:
+      - names: [replication]
+
+# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md
+worker_app: synapse.app.homeserver
+
+# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
+# reduce the main worker's offerings to core homeserver business
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list  %}
+notify_appservices: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list  %}
+send_federation: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list  %}
+enable_media_repo: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list  %}
+start_pushers: false
+{% endif %}
+{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list  %}
+update_user_directory: false
+{% endif %}
+
+# rather let systemd handle the forking
+daemonize: false
+{% endif %}
 
 ## Homeserver blocking ##
 

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2

@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+# c.f. https://github.com/matrix-org/synapse/pull/4662
+[Unit]
+Description=Synapse Matrix Worker
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+Type=simple
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre=/bin/sleep 5
+
+# systemd ftw 🤦‍♂️
+# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
+ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
+			exec /usr/bin/docker exec \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			matrix-synapse \
+			python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
+
+ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i
+KillMode=process
+Restart=always
+RestartSec=10
+SyslogIdentifier=matrix-synapse-%i
+
+[Install]
+WantedBy=matrix-synapse.service

roles/matrix-synapse/templates/synapse/worker.yaml.j2

@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+worker_app: synapse.app.{{ item.worker }}
+
+worker_replication_host: 127.0.0.1
+worker_replication_port: {{ matrix_synapse_replication_tcp_port }}
+worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
+
+{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
+worker_listeners:
+  - type: http
+    port: {{ item.port }}
+    resources:
+      - names:
+{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %}
+        - client
+{% elif item.worker in [ 'federation_reader' ] %}
+        - federation
+{% elif item.worker in [ 'media_repository' ] %}
+        - media
+{% endif %}
+{% endif %}
+
+{% if item.worker == 'frontend_proxy' %}
+worker_main_http_uri: http://127.0.0.1:8008
+{% endif %}
+  
+worker_daemonize: false
+worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
+worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config